v0.14.0

Prevent messages from being sent if other messages have failed to send

.babelrc

@@ -1,3 +1,13 @@
 {
-  "presets": ["es2015"]
+    "presets": ["es2015"],
+    "plugins": [
+        // this transforms async functions into generator functions, which
+        // are then made to use the regenerator module by babel's
+        // transform-regnerator plugin (which is enabled by es2015).
+        "transform-async-to-bluebird",
+
+        // This makes sure that the regenerator runtime is available to
+        // the transpiled code.
+        "transform-runtime",
+    ],
 }

.editorconfig

@@ -0,0 +1,23 @@
+#   Copyright 2017 Aviral Dasgupta
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+root = true
+
+[*]
+charset=utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+trim_trailing_whitespace = true

.eslintrc.js

@@ -1,4 +1,5 @@
 module.exports = {
+    parser: "babel-eslint",
     parserOptions: {
         ecmaVersion: 6,
         sourceType: "module",
@@ -8,15 +9,27 @@ module.exports = {
     env: {
         browser: true,
         node: true,
+
+        // babel's transform-runtime converts references to ES6 globals such as
+        // Promise and Map to core-js polyfills, so we can use ES6 globals.
+        es6: true,
     },
     extends: ["eslint:recommended", "google"],
     rules: {
-        // rules we've always adhered to
+        // rules we've always adhered to or now do
         "max-len": ["error", {
             code: 90,
             ignoreComments: true,
         }],
         curly: ["error", "multi-line"],
+        "prefer-const": ["error"],
+        "comma-dangle": ["error", {
+            arrays: "always-multiline",
+            objects: "always-multiline",
+            imports: "always-multiline",
+            exports: "always-multiline",
+            functions: "always-multiline",
+        }],
 
         // loosen jsdoc requirements a little
         "require-jsdoc": ["error", {
@@ -47,8 +60,7 @@ module.exports = {
         // we set these to warnings, and assert that the number
         // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "comma-dangle": ["warn"],
-        "brace-style": ["warn"],
+        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],

.gitignore

@@ -1,4 +1,6 @@
-.jsdoc
+/.jsdocbuild
+/.jsdoc
+
 node_modules
 .lock-wscript
 build/Release
@@ -8,6 +10,7 @@ out
 reports
 /dist
 /lib
+/specbuild
 
 # version file and tarball created by 'npm pack'
 /git-revision.txt

.travis.yml

@@ -1,3 +1,5 @@
 language: node_js
 node_js:
     - node # Latest stable version of nodejs.
+script:
+    - ./travis.sh

CONTRIBUTING.rst

@@ -1,4 +1,115 @@
 Contributing code to matrix-js-sdk
 ==================================
 
-matrix-js-sdk follows the same pattern as https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst
+Everyone is welcome to contribute code to matrix-js-sdk, provided that they are
+willing to license their contributions under the same license as the project
+itself. We follow a simple 'inbound=outbound' model for contributions: the act
+of submitting an 'inbound' contribution means that the contributor agrees to
+license the code under the same terms as the project's overall 'outbound'
+license - in this case, Apache Software License v2 (see `<LICENSE>`_).
+
+How to contribute
+~~~~~~~~~~~~~~~~~
+
+The preferred and easiest way to contribute changes to the project is to fork
+it on github, and then create a pull request to ask us to pull your changes
+into our repo (https://help.github.com/articles/using-pull-requests/)
+
+**The single biggest thing you need to know is: please base your changes on
+the develop branch - /not/ master.**
+
+We use the master branch to track the most recent release, so that folks who
+blindly clone the repo and automatically check out master get something that
+works. Develop is the unstable branch where all the development actually
+happens: the workflow is that contributors should fork the develop branch to
+make a 'feature' branch for a particular contribution, and then make a pull
+request to merge this back into the matrix.org 'official' develop branch. We
+use github's pull request workflow to review the contribution, and either ask
+you to make any refinements needed or merge it and make them ourselves. The
+changes will then land on master when we next do a release.
+
+We use Travis for continuous integration, and all pull requests get
+automatically tested by Travis: if your change breaks the build, then the PR
+will show that there are failed checks, so please check back after a few
+minutes.
+
+Code style
+~~~~~~~~~~
+
+The code-style for matrix-js-sdk is not formally documented, but contributors
+are encouraged to read the code style document for matrix-react-sdk
+(`<https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md>`_)
+and follow the principles set out there.
+
+Please ensure your changes match the cosmetic style of the existing project,
+and **never** mix cosmetic and functional changes in the same commit, as it
+makes it horribly hard to review otherwise.
+
+Attribution
+~~~~~~~~~~~
+
+Everyone who contributes anything to Matrix is welcome to be listed in the
+AUTHORS.rst file for the project in question. Please feel free to include a
+change to AUTHORS.rst in your pull request to list yourself and a short
+description of the area(s) you've worked on. Also, we sometimes have swag to
+give away to contributors - if you feel that Matrix-branded apparel is missing
+from your life, please mail us your shipping address to matrix at matrix.org
+and we'll try to fix it :)
+
+Sign off
+~~~~~~~~
+
+In order to have a concrete record that your contribution is intentional
+and you agree to license it under the same terms as the project's license, we've adopted the
+same lightweight approach that the Linux Kernel
+(https://www.kernel.org/doc/Documentation/SubmittingPatches), Docker
+(https://github.com/docker/docker/blob/master/CONTRIBUTING.md), and many other
+projects use: the DCO (Developer Certificate of Origin:
+http://developercertificate.org/). This is a simple declaration that you wrote
+the contribution or otherwise have the right to contribute it to Matrix::
+
+    Developer Certificate of Origin
+    Version 1.1
+
+    Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+    660 York Street, Suite 102,
+    San Francisco, CA 94110 USA
+
+    Everyone is permitted to copy and distribute verbatim copies of this
+    license document, but changing it is not allowed.
+
+    Developer's Certificate of Origin 1.1
+
+    By making a contribution to this project, I certify that:
+
+    (a) The contribution was created in whole or in part by me and I
+        have the right to submit it under the open source license
+        indicated in the file; or
+
+    (b) The contribution is based upon previous work that, to the best
+        of my knowledge, is covered under an appropriate open source
+        license and I have the right under that license to submit that
+        work with modifications, whether created in whole or in part
+        by me, under the same open source license (unless I am
+        permitted to submit under a different license), as indicated
+        in the file; or
+
+    (c) The contribution was provided directly to me by some other
+        person who certified (a), (b) or (c) and I have not modified
+        it.
+
+    (d) I understand and agree that this project and the contribution
+        are public and that a record of the contribution (including all
+        personal information I submit with it, including my sign-off) is
+        maintained indefinitely and may be redistributed consistent with
+        this project or the open source license(s) involved.
+
+If you agree to this for your contribution, then all that's needed is to
+include the line in your commit or pull request comment::
+
+    Signed-off-by: Your Name <your@email.example.org>
+
+...using your real name; unfortunately pseudonyms and anonymous contributions
+can't be accepted. Git makes this trivial - just use the -s flag when you do
+``git commit``, having first set ``user.name`` and ``user.email`` git configs
+(which you should have done anyway :)

README.md

@@ -13,7 +13,8 @@ In a browser
 Download either the full or minified version from
 https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
 ``<script>`` to your page. There will be a global variable ``matrixcs``
-attached to ``window`` through which you can access the SDK.
+attached to ``window`` through which you can access the SDK. See below for how to
+include libolm to enable end-to-end-encryption.
 
 Please check [the working browser example](examples/browser) for more information.
 
@@ -30,7 +31,60 @@ In Node.js
   });
 ```
 
-Please check [the Node.js terminal app](examples/node) for a more complex example.
+See below for how to include libolm to enable end-to-end-encryption. Please check
+[the Node.js terminal app](examples/node) for a more complex example.
+
+To start the client:
+
+```javascript
+await client.startClient({initialSyncLimit: 10});
+```
+
+You can perform a call to `/sync` to get the current state of the client:
+
+```javascript
+client.once('sync', function(state, prevState, res) {
+    if(state === 'PREPARED') {
+        console.log("prepared");
+    } else {
+        console.log(state);
+        process.exit(1);
+    }
+});
+```
+
+To send a message:
+
+```javascript
+var content = {
+    "body": "message text",
+    "msgtype": "m.text"
+};
+client.sendEvent("roomId", "m.room.message", content, "", (err, res) => {
+    console.log(err);
+});
+```
+
+To listen for message events:
+
+```javascript
+client.on("Room.timeline", function(event, room, toStartOfTimeline) {
+  if (event.getType() !== "m.room.message") {
+    return; // only use messages
+  }
+  console.log(event.event.content.body);
+});
+```
+
+By default, the `matrix-js-sdk` client uses the `MatrixInMemoryStore` to store events as they are received. For example to iterate through the currently stored timeline for a room:
+
+```javascript
+Object.keys(client.store.rooms).forEach((roomId) => {
+  client.getRoom(roomId).timeline.forEach(t => {
+      console.log(t.event);
+  });
+});
+```
 
 What does this SDK do?
 ----------------------
@@ -65,6 +119,7 @@ Later versions of the SDK will:
 Usage
 =====
 
+
 Conventions
 -----------
 
@@ -235,6 +290,46 @@ host the API reference from the source files like this:
 
 Then visit ``http://localhost:8005`` to see the API docs.
 
+End-to-end encryption support
+=============================
+
+The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
+[libolm](http://matrix.org/git/olm). It is left up to the application to make
+libolm available, via the ``Olm`` global.
+
+It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
+initialise the crypto layer.
+
+If the ``Olm`` global is not available, the SDK will show a warning, as shown
+below; ``initCrypto()`` will also fail.
+
+```
+Unable to load crypto module: crypto will be disabled: Error: global.Olm is not defined
+```
+
+If the crypto layer is not (successfully) initialised, the SDK will continue to
+work for unencrypted rooms, but it will not support the E2E parts of the Matrix
+specification.
+
+To provide the Olm library in a browser application:
+
+ * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
+ * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
+ 
+To provide the Olm library in a node.js application:
+
+ * ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``
+   (replace the URL with the latest version you want to use from
+    https://matrix.org/packages/npm/olm/)
+ * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
+
+If you want to package Olm as dependency for your node.js application, you
+can use ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz
+--save-optional`` (if your application also works without e2e crypto enabled)
+or ``--save`` (if it doesn't) to do so.
+
+
 Contributing
 ============
 *This section is for people who want to modify the SDK. If you just

browser-index.js

@@ -1,4 +1,35 @@
 var matrixcs = require("./lib/matrix");
-matrixcs.request(require("browser-request"));
+const request = require('browser-request');
+const queryString = require('qs');
+
+matrixcs.request(function(opts, fn) {
+    // We manually fix the query string for browser-request because
+    // it doesn't correctly handle cases like ?via=one&via=two. Instead
+    // we mimic `request`'s query string interface to make it all work
+    // as expected.
+    // browser-request will happily take the constructed string as the
+    // query string without trying to modify it further.
+    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
+    return request(opts, fn);
+});
+
+// just *accessing* indexedDB throws an exception in firefox with
+// indexeddb disabled.
+var indexedDB;
+try {
+    indexedDB = global.indexedDB;
+} catch(e) {}
+
+// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
+if (indexedDB) {
+    matrixcs.setCryptoStoreFactory(
+        function() {
+            return new matrixcs.IndexedDBCryptoStore(
+                indexedDB, "matrix-js-sdk:crypto"
+            );
+        }
+    );
+}
+
 module.exports = matrixcs; // keep export for browserify package deps
 global.matrixcs = matrixcs;

docs/warning-on-unverified-devices.txt

@@ -0,0 +1,31 @@
+Random notes from Matthew on the two possible approaches for warning users about unexpected
+unverified devices popping up in their rooms....
+
+Original idea...
+================
+
+Warn when an existing user adds an unknown device to a room.
+
+Warn when a user joins the room with unverified or unknown devices.
+
+Warn when you initial sync if the room has any unverified devices in it.
+ ^ this is good enough if we're doing local storage.
+ OR, better:
+Warn when you initial sync if the room has any new undefined devices since you were last there.
+ => This means persisting the rooms that devices are in, across initial syncs.
+
+
+Updated idea...
+===============
+
+Warn when the user tries to send a message:
+  - If the room has unverified devices which the user has not yet been told about in the context of this room
+    ...or in the context of this user?  currently all verification is per-user, not per-room.
+    ...this should be good enough.
+
+  - so track whether we have warned the user or not about unverified devices - blocked, unverified, verified, unverified_warned.
+    throw an error when trying to encrypt if there are pure unverified devices there
+    app will have to search for the devices which are pure unverified to warn about them - have to do this from MembersList anyway?
+      - or megolm could warn which devices are causing the problems.
+
+Why do we wait to establish outbound sessions?  It just makes a horrible pause when we first try to send a message... but could otherwise unnecessarily consume resources?

examples/node/app.js

@@ -202,9 +202,9 @@ function printRoomList() {
             dateStr = new Date(msg.getTs()).toISOString().replace(
                 /T/, ' ').replace(/\..+/, '');
         }
-        var me = roomList[i].getMember(myUserId);
-        if (me) {
-            fmt = fmts[me.membership];
+        var myMembership = roomList[i].getMyMembership();
+        if (myMembership) {
+            fmt = fmts[myMembership];
         }
         var roomName = fixWidth(roomList[i].name, 25);
         print(

examples/voip/browserTest.js

@@ -44,7 +44,7 @@ window.onload = function() {
     disableButtons(true, true, true);
 };
 
-matrixClient.on("sync", function(state, prevState, data) {
+client.on("sync", function(state, prevState, data) {
     switch (state) {
         case "PREPARED":
           syncComplete();

jenkins.sh

@@ -1,9 +1,12 @@
 #!/bin/bash -l
 
-export NVM_DIR="/home/jenkins/.nvm"
+set -x
+
+export NVM_DIR="$HOME/.nvm"
 [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
-nvm use 4
-npm install
+
+nvm use 6 || exit $?
+npm install || exit $?
 
 RC=0
 
@@ -12,6 +15,9 @@ function fail {
     RC=1
 }
 
+# don't use last time's test reports
+rm -rf reports coverage || exit $?
+
 npm test || fail "npm test finished with return code $?"
 
 npm run -s lint -- -f checkstyle > eslint.xml ||

package.json

@@ -1,20 +1,27 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.7.4",
+  "version": "0.14.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {
-    "test": "istanbul cover --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" jasmine-node -- spec --verbose --junitreport --captureExceptions",
-    "check": "jasmine-node spec --verbose --junitreport --captureExceptions",
-    "gendoc": "jsdoc -r lib -P package.json -R README.md -d .jsdoc",
-    "start": "babel -s -w -d lib src",
-    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify --exclude olm browser-index.js -o dist/browser-matrix.js --ignore-missing && uglifyjs -c -m -o dist/browser-matrix.min.js dist/browser-matrix.js",
+    "test:build": "babel -s -d specbuild spec",
+    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" node_modules/mocha/bin/_mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
+    "test": "npm run test:build && npm run test:run",
+    "check": "npm run test:build && _mocha --recursive specbuild --colors",
+    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
+    "start": "npm run start:init && npm run start:watch",
+    "start:watch": "babel -s -w --skip-initial-build -d lib src",
+    "start:init": "babel -s -d lib src",
+    "clean": "rimraf lib dist",
+    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
     "dist": "npm run build",
-    "watch": "watchify --exclude olm browser-index.js -o dist/browser-matrix-dev.js -v",
-    "lint": "eslint --max-warnings 3480 src spec",
-    "prepublish": "npm run lint && npm run build && git rev-parse HEAD > git-revision.txt"
+    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
+    "lint": "eslint --max-warnings 102 src spec",
+    "prepublish": "npm run clean && npm run build && git rev-parse HEAD > git-revision.txt"
   },
   "repository": {
+    "type": "git",
     "url": "https://github.com/matrix-org/matrix-js-sdk"
   },
   "keywords": [
@@ -24,6 +31,7 @@
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
+    ".babelrc",
     ".eslintrc.js",
     "spec/.eslintrc.js",
     "CHANGELOG.md",
@@ -45,24 +53,41 @@
   ],
   "dependencies": {
     "another-json": "^0.2.0",
+    "babel-runtime": "^6.26.0",
+    "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
-    "browserify": "^10.2.3",
-    "q": "^1.4.1",
-    "request": "^2.53.0"
+    "content-type": "^1.0.2",
+    "loglevel": "1.6.1",
+    "qs": "^6.5.2",
+    "request": "^2.88.0"
   },
   "devDependencies": {
     "babel-cli": "^6.18.0",
+    "babel-eslint": "^8.1.1",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
+    "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-es2015": "^6.18.0",
+    "browserify": "^16.2.3",
+    "browserify-shim": "^3.8.13",
     "eslint": "^3.13.1",
     "eslint-config-google": "^0.7.1",
-    "istanbul": "^0.3.13",
-    "jasmine-node": "^1.14.5",
-    "jsdoc": "^3.4.0",
+    "exorcist": "^0.4.0",
+    "expect": "^1.20.2",
+    "istanbul": "^0.4.5",
+    "jsdoc": "^3.5.5",
+    "lolex": "^1.5.2",
+    "matrix-mock-request": "^1.2.2",
+    "mocha": "^5.2.0",
+    "mocha-jenkins-reporter": "^0.4.0",
     "rimraf": "^2.5.4",
-    "uglifyjs": "^2.4.10",
-    "watchify": "^3.2.1"
+    "source-map-support": "^0.4.11",
+    "sourceify": "^0.1.0",
+    "uglify-js": "^2.8.26",
+    "watchify": "^3.11.0"
   },
-  "optionalDependencies": {
-    "olm": "https://matrix.org/packages/npm/olm/olm-2.1.0.tgz"
+  "browserify": {
+    "transform": [
+      "sourceify"
+    ]
   }
 }

release.sh

@@ -178,10 +178,6 @@ if [ $dodist -eq 0 ]; then
     done
 fi
 
-# push the release branch (github can't release from
-# a branch it doesn't have)
-git push origin "$rel_branch"
-
 if [ -n "$signing_id" ]; then
     # make a signed tag
     # gnupg seems to fail to get the right tty device unless we set it here
@@ -190,8 +186,55 @@ else
     git tag -a -F "${latest_changes}" "$tag"
 fi
 
-# push the tag
-git push origin "$tag"
+# push the tag and the release branch
+git push origin "$rel_branch" "$tag"
+
+if [ -n "$signing_id" ]; then
+    # make a signature for the source tarball.
+    #
+    # github will make us a tarball from the tag - we want to create a
+    # signature for it, which means that first of all we need to check that
+    # it's correct.
+    #
+    # we can't deterministically build exactly the same tarball, due to
+    # differences in gzip implementation - but we *can* build the same tar - so
+    # the easiest way to check the validity of the tarball from git is to unzip
+    # it and compare it with our own idea of what the tar should look like.
+
+    # the name of the sig file we want to create
+    source_sigfile="${tag}-src.tar.gz.asc"
+
+    tarfile="$tag.tar.gz"
+    gh_project_url=$(git remote get-url origin |
+                            sed -e 's#^git@github\.com:#https://github.com/#' \
+                                -e 's#^git\+ssh://git@github\.com/#https://github.com/#' \
+                                -e 's/\.git$//')
+    project_name="${gh_project_url##*/}"
+    curl -L "${gh_project_url}/archive/${tarfile}" -o "${tarfile}"
+
+    # unzip it and compare it with the tar we would generate
+    if ! cmp --silent <(gunzip -c $tarfile) \
+         <(git archive --format tar --prefix="${project_name}-${release}/" "$tag"); then
+
+        # we don't bail out here, because really it's more likely that our comparison
+        # screwed up and it's super annoying to abort the script at this point.
+        cat >&2 <<EOF
+!!!!!!!!!!!!!!!!!
+!!!! WARNING !!!!
+
+Mismatch between our own tarfile and that generated by github: not signing
+source tarball.
+
+To resolve, determine if $tarfile is correct, and if so sign it with gpg and
+attach it to the release as $source_sigfile.
+
+!!!!!!!!!!!!!!!!!
+EOF
+    else
+        gpg -u "$signing_id" --armor --output "$source_sigfile" --detach-sig "$tarfile"
+        assets="$assets -a $source_sigfile"
+    fi
+fi
 
 hubflags=''
 if [ $prerelease -eq 1 ]; then
@@ -202,7 +245,7 @@ release_text=`mktemp`
 echo "$tag" > "${release_text}"
 echo >> "${release_text}"
 cat "${latest_changes}" >> "${release_text}"
-hub release create $hubflags $assets -f "${release_text}" "$tag"
+hub release create $hubflags $assets -F "${release_text}" "$tag"
 
 if [ $dodist -eq 0 ]; then
     rm -rf "$builddir"
@@ -210,6 +253,9 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"
 
+# publish to npmjs
+npm publish
+
 if [ -z "$skip_jsdoc" ]; then
     echo "generating jsdocs"
     npm run gendoc
@@ -225,11 +271,17 @@ if [ -z "$skip_jsdoc" ]; then
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
 fi
 
+# if it is a pre-release, leave it on the release branch for now.
+if [ $prerelease -eq 1 ]; then
+    git checkout "$rel_branch"
+    exit 0
+fi
+
 # merge release branch to master
 echo "updating master branch"
 git checkout master
 git pull
-git merge --ff-only "$rel_branch"
+git merge "$rel_branch"
 
 # push master  and docs (if generated) to github
 git push origin master
@@ -237,9 +289,6 @@ if [ -z "$skip_jsdoc" ]; then
     git push origin gh-pages
 fi
 
-# publish to npmjs
-npm publish
-
 # finally, merge master back onto develop
 git checkout develop
 git pull

spec/.eslintrc.js

@@ -1,5 +1,5 @@
 module.exports = {
     env: {
-        jasmine: true,
+        mocha: true,
     },
 }

spec/MockStorageApi.js

@@ -40,14 +40,16 @@ MockStorageApi.prototype = {
         return this.keys[index];
     },
     _recalc: function() {
-        var keys = [];
-        for (var k in this.data) {
-            if (!this.data.hasOwnProperty(k)) { continue; }
+        const keys = [];
+        for (const k in this.data) {
+            if (!this.data.hasOwnProperty(k)) {
+                continue;
+            }
             keys.push(k);
         }
         this.keys = keys;
         this.length = keys.length;
-    }
+    },
 };
 
 /** */

spec/TestClient.js

@@ -0,0 +1,225 @@
+/*
+Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+
+// load olm before the sdk if possible
+import './olm-loader';
+
+import sdk from '..';
+import testUtils from './test-utils';
+import MockHttpBackend from 'matrix-mock-request';
+import expect from 'expect';
+import Promise from 'bluebird';
+import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';
+
+/**
+ * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
+ *
+ * @constructor
+ * @param {string} userId
+ * @param {string} deviceId
+ * @param {string} accessToken
+ *
+ * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
+ *     session store. If undefined, we will create a MockStorageApi.
+ */
+export default function TestClient(
+    userId, deviceId, accessToken, sessionStoreBackend,
+) {
+    this.userId = userId;
+    this.deviceId = deviceId;
+
+    if (sessionStoreBackend === undefined) {
+        sessionStoreBackend = new testUtils.MockStorageApi();
+    }
+    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
+
+    // expose this so the tests can get to it
+    this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+
+    this.httpBackend = new MockHttpBackend();
+    this.client = sdk.createClient({
+        baseUrl: "http://" + userId + ".test.server",
+        userId: userId,
+        accessToken: accessToken,
+        deviceId: deviceId,
+        sessionStore: sessionStore,
+        cryptoStore: this.cryptoStore,
+        request: this.httpBackend.requestFn,
+    });
+
+    this.deviceKeys = null;
+    this.oneTimeKeys = {};
+}
+
+TestClient.prototype.toString = function() {
+    return 'TestClient[' + this.userId + ']';
+};
+
+/**
+ * start the client, and wait for it to initialise.
+ *
+ * @return {Promise}
+ */
+TestClient.prototype.start = function() {
+    console.log(this + ': starting');
+    this.httpBackend.when("GET", "/pushrules").respond(200, {});
+    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+    this.expectDeviceKeyUpload();
+
+    // we let the client do a very basic initial sync, which it needs before
+    // it will upload one-time keys.
+    this.httpBackend.when("GET", "/sync").respond(200, { next_batch: 1 });
+
+    this.client.startClient({
+        // set this so that we can get hold of failed events
+        pendingEventOrdering: 'detached',
+    });
+
+    return Promise.all([
+        this.httpBackend.flushAllExpected(),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
+        console.log(this + ': started');
+    });
+};
+
+/**
+ * stop the client
+ * @return {Promise} Resolves once the mock http backend has finished all pending flushes
+ */
+TestClient.prototype.stop = function() {
+    this.client.stopClient();
+    return this.httpBackend.stop();
+};
+
+/**
+ * Set up expectations that the client will upload device keys.
+ */
+TestClient.prototype.expectDeviceKeyUpload = function() {
+    const self = this;
+    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
+        expect(content.one_time_keys).toBe(undefined);
+        expect(content.device_keys).toBeTruthy();
+
+        console.log(self + ': received device keys');
+        // we expect this to happen before any one-time keys are uploaded.
+        expect(Object.keys(self.oneTimeKeys).length).toEqual(0);
+
+        self.deviceKeys = content.device_keys;
+        return {one_time_key_counts: {signed_curve25519: 0}};
+    });
+};
+
+
+/**
+ * If one-time keys have already been uploaded, return them. Otherwise,
+ * set up an expectation that the keys will be uploaded, and wait for
+ * that to happen.
+ *
+ * @returns {Promise} for the one-time keys
+ */
+TestClient.prototype.awaitOneTimeKeyUpload = function() {
+    if (Object.keys(this.oneTimeKeys).length != 0) {
+        // already got one-time keys
+        return Promise.resolve(this.oneTimeKeys);
+    }
+
+    this.httpBackend.when("POST", "/keys/upload")
+        .respond(200, (path, content) => {
+            expect(content.device_keys).toBe(undefined);
+            expect(content.one_time_keys).toBe(undefined);
+            return {one_time_key_counts: {
+                signed_curve25519: Object.keys(this.oneTimeKeys).length,
+            }};
+        });
+
+    this.httpBackend.when("POST", "/keys/upload")
+          .respond(200, (path, content) => {
+              expect(content.device_keys).toBe(undefined);
+              expect(content.one_time_keys).toBeTruthy();
+              expect(content.one_time_keys).toNotEqual({});
+              console.log('%s: received %i one-time keys', this,
+                          Object.keys(content.one_time_keys).length);
+              this.oneTimeKeys = content.one_time_keys;
+              return {one_time_key_counts: {
+                  signed_curve25519: Object.keys(this.oneTimeKeys).length,
+              }};
+          });
+
+    // this can take ages
+    return this.httpBackend.flush('/keys/upload', 2, 1000).then((flushed) => {
+        expect(flushed).toEqual(2);
+        return this.oneTimeKeys;
+    });
+};
+
+/**
+ * Set up expectations that the client will query device keys.
+ *
+ * We check that the query contains each of the users in `response`.
+ *
+ * @param {Object} response   response to the query.
+ */
+TestClient.prototype.expectKeyQuery = function(response) {
+    this.httpBackend.when('POST', '/keys/query').respond(
+        200, (path, content) => {
+            Object.keys(response.device_keys).forEach((userId) => {
+                expect(content.device_keys[userId]).toEqual({});
+            });
+            return response;
+        });
+};
+
+
+/**
+ * get the uploaded curve25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getDeviceKey = function() {
+    const keyId = 'curve25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};
+
+
+/**
+ * get the uploaded ed25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getSigningKey = function() {
+    const keyId = 'ed25519:' + this.deviceId;
+    return this.deviceKeys.keys[keyId];
+};
+
+/**
+ * flush a single /sync request, and wait for the syncing event
+ *
+ * @returns {Promise} promise which completes once the sync has been flushed
+ */
+TestClient.prototype.flushSync = function() {
+    console.log(`${this}: flushSync`);
+    return Promise.all([
+        this.httpBackend.flush('/sync', 1),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
+        console.log(`${this}: flushSync completed`);
+    });
+};

spec/integ/devicelist-integ-spec.js

@@ -0,0 +1,400 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import TestClient from '../TestClient';
+import testUtils from '../test-utils';
+
+const ROOM_ID = "!room:id";
+
+/**
+ * get a /sync response which contains a single e2e room (ROOM_ID), with the
+ * members given
+ *
+ * @param {string[]} roomMembers
+ *
+ * @return {object} sync response
+ */
+function getSyncResponse(roomMembers) {
+    const stateEvents = [
+        testUtils.mkEvent({
+            type: 'm.room.encryption',
+            skey: '',
+            content: {
+                algorithm: 'm.megolm.v1.aes-sha2',
+            },
+        }),
+    ];
+
+    Array.prototype.push.apply(
+        stateEvents,
+        roomMembers.map(
+            (m) => testUtils.mkMembership({
+                mship: 'join',
+                sender: m,
+            }),
+        ),
+    );
+
+    const syncResponse = {
+        next_batch: 1,
+        rooms: {
+            join: {
+                [ROOM_ID]: {
+                    state: {
+                        events: stateEvents,
+                    },
+                },
+            },
+        },
+    };
+
+    return syncResponse;
+}
+
+
+describe("DeviceList management:", function() {
+    if (!global.Olm) {
+        console.warn('not running deviceList tests: Olm not present');
+        return;
+    }
+
+    let sessionStoreBackend;
+    let aliceTestClient;
+
+    async function createTestClient() {
+        const testClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs", sessionStoreBackend,
+        );
+        await testClient.client.initCrypto();
+        return testClient;
+    }
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        // we create our own sessionStoreBackend so that we can use it for
+        // another TestClient.
+        sessionStoreBackend = new testUtils.MockStorageApi();
+
+        aliceTestClient = await createTestClient();
+    });
+
+    afterEach(function() {
+        return aliceTestClient.stop();
+    });
+
+    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
+        return aliceTestClient.start().then(function() {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+            return aliceTestClient.flushSync();
+        }).then(function() {
+            console.log("Forcing alice to download our device keys");
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {},
+                },
+            });
+
+            return Promise.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log("Telling alice to send a megolm message");
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/',
+            ).respond(200, {
+                    event_id: '$event_id',
+            });
+
+            return Promise.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
+            ]);
+        });
+    });
+
+
+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.flushSync();
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return Promise.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1),
+                   ),
+                   aliceTestClient.client._crypto._deviceList.saveIfDirty(),
+               ]);
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   expect(data.syncToken).toEqual(1);
+               });
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               // flush both syncs
+               return aliceTestClient.flushSync().then(() => {
+                   return aliceTestClient.flushSync();
+               });
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   if (bobStat != 1 && bobStat != 2) {
+                       throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                       bobStat);
+                   }
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
+                       );
+                   }
+               });
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   expect(bobStat).toEqual(3);
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
+                       );
+                   }
+               });
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   const chrisStat = data.trackingStatus['@bob:xyz'];
+
+                   expect(bobStat).toEqual(3);
+                   expect(chrisStat).toEqual(3);
+                   expect(data.syncToken).toEqual(3);
+               });
+           });
+       }).timeout(3000);
+
+    // https://github.com/vector-im/riot-web/issues/4983
+    describe("Alice should know she has stale device lists", () => {
+        beforeEach(async function() {
+            await aliceTestClient.start();
+
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, getSyncResponse(['@bob:xyz']));
+            await aliceTestClient.flushSync();
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                200, {
+                    device_keys: {
+                        '@bob:xyz': {},
+                    },
+                },
+            );
+            await aliceTestClient.httpBackend.flush('/keys/query', 1);
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toBeGreaterThan(
+                    0, "Alice should be tracking bob's device list",
+                );
+            });
+        });
+
+        it("when Bob leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        join: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Alice leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        leave: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Bob leaves whilst Alice is offline", async function() {
+            aliceTestClient.stop();
+
+            const anotherTestClient = await createTestClient();
+
+            try {
+                await anotherTestClient.start();
+                anotherTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, getSyncResponse([]));
+                await anotherTestClient.flushSync();
+                await anotherTestClient.client._crypto._deviceList.saveIfDirty();
+
+                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                    const bobStat = data.trackingStatus['@bob:xyz'];
+
+                    expect(bobStat).toEqual(
+                        0, "Alice should have marked bob's device list as untracked",
+                    );
+                });
+            } finally {
+                anotherTestClient.stop();
+            }
+        });
+    });
+});

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,13 +1,18 @@
 "use strict";
-var sdk = require("../..");
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+
+import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient events", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend;
-    var selfUserId = "@alice:localhost";
-    var selfAccessToken = "aseukfgwef";
+    const baseUrl = "http://localhost.or.something";
+    let client;
+    let httpBackend;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -16,7 +21,7 @@ describe("MatrixClient events", function() {
         client = sdk.createClient({
             baseUrl: baseUrl,
             userId: selfUserId,
-            accessToken: selfAccessToken
+            accessToken: selfAccessToken,
         });
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
@@ -25,17 +30,18 @@ describe("MatrixClient events", function() {
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
         client.stopClient();
+        return httpBackend.stop();
     });
 
     describe("emissions", function() {
-        var SYNC_DATA = {
+        const SYNC_DATA = {
             next_batch: "s_5_3",
             presence: {
                 events: [
                     utils.mkPresence({
-                        user: "@foo:bar", name: "Foo Bar", presence: "online"
-                    })
-                ]
+                        user: "@foo:bar", name: "Foo Bar", presence: "online",
+                    }),
+                ],
             },
             rooms: {
                 join: {
@@ -43,30 +49,30 @@ describe("MatrixClient events", function() {
                         timeline: {
                             events: [
                                 utils.mkMessage({
-                                    room: "!erufh:bar", user: "@foo:bar", msg: "hmmm"
-                                })
+                                    room: "!erufh:bar", user: "@foo:bar", msg: "hmmm",
+                                }),
                             ],
-                            prev_batch: "s"
+                            prev_batch: "s",
                         },
                         state: {
                             events: [
                                 utils.mkMembership({
-                                    room: "!erufh:bar", mship: "join", user: "@foo:bar"
+                                    room: "!erufh:bar", mship: "join", user: "@foo:bar",
                                 }),
                                 utils.mkEvent({
                                     type: "m.room.create", room: "!erufh:bar",
                                     user: "@foo:bar",
                                     content: {
-                                        creator: "@foo:bar"
-                                    }
-                                })
-                            ]
-                        }
-                    }
-                }
-            }
+                                        creator: "@foo:bar",
+                                    },
+                                }),
+                            ],
+                        },
+                    },
+                },
+            },
         };
-        var NEXT_SYNC_DATA = {
+        const NEXT_SYNC_DATA = {
             next_batch: "e_6_7",
             rooms: {
                 join: {
@@ -74,44 +80,45 @@ describe("MatrixClient events", function() {
                         timeline: {
                             events: [
                                 utils.mkMessage({
-                                    room: "!erufh:bar", user: "@foo:bar", msg: "ello ello"
+                                    room: "!erufh:bar", user: "@foo:bar",
+                                    msg: "ello ello",
                                 }),
                                 utils.mkMessage({
-                                    room: "!erufh:bar", user: "@foo:bar", msg: ":D"
+                                    room: "!erufh:bar", user: "@foo:bar", msg: ":D",
                                 }),
-                            ]
+                            ],
                         },
                         ephemeral: {
                             events: [
                                 utils.mkEvent({
                                     type: "m.typing", room: "!erufh:bar", content: {
-                                        user_ids: ["@foo:bar"]
-                                    }
-                                })
-                            ]
-                        }
-                    }
-                }
-            }
+                                        user_ids: ["@foo:bar"],
+                                    },
+                                }),
+                            ],
+                        },
+                    },
+                },
+            },
         };
 
         it("should emit events from both the first and subsequent /sync calls",
-        function(done) {
+        function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
-            var expectedEvents = [];
+            let expectedEvents = [];
             expectedEvents = expectedEvents.concat(
                 SYNC_DATA.presence.events,
                 SYNC_DATA.rooms.join["!erufh:bar"].timeline.events,
                 SYNC_DATA.rooms.join["!erufh:bar"].state.events,
                 NEXT_SYNC_DATA.rooms.join["!erufh:bar"].timeline.events,
-                NEXT_SYNC_DATA.rooms.join["!erufh:bar"].ephemeral.events
+                NEXT_SYNC_DATA.rooms.join["!erufh:bar"].ephemeral.events,
             );
 
             client.on("event", function(event) {
-                var found = false;
-                for (var i = 0; i < expectedEvents.length; i++) {
+                let found = false;
+                for (let i = 0; i < expectedEvents.length; i++) {
                     if (expectedEvents[i].event_id === event.getId()) {
                         expectedEvents.splice(i, 1);
                         found = true;
@@ -119,49 +126,56 @@ describe("MatrixClient events", function() {
                     }
                 }
                 expect(found).toBe(
-                    true, "Unexpected 'event' emitted: " + event.getType()
+                    true, "Unexpected 'event' emitted: " + event.getType(),
                 );
             });
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                // wait for two SYNCING events
+                utils.syncPromise(client).then(() => {
+                    return utils.syncPromise(client);
+                }),
+                httpBackend.flushAllExpected(),
+            ]).then(() => {
                 expect(expectedEvents.length).toEqual(
-                    0, "Failed to see all events from /sync calls"
+                    0, "Failed to see all events from /sync calls",
                 );
-                done();
             });
         });
 
         it("should emit User events", function(done) {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-            var fired = false;
+            let fired = false;
             client.on("User.presence", function(event, user) {
                 fired = true;
-                expect(user).toBeDefined();
-                expect(event).toBeDefined();
-                if (!user || !event) { return; }
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
+                if (!user || !event) {
+                    return;
+                }
 
                 expect(event.event).toEqual(SYNC_DATA.presence.events[0]);
                 expect(user.presence).toEqual(
-                    SYNC_DATA.presence.events[0].content.presence
+                    SYNC_DATA.presence.events[0].content.presence,
                 );
             });
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 expect(fired).toBe(true, "User.presence didn't fire.");
                 done();
             });
         });
 
-        it("should emit Room events", function(done) {
+        it("should emit Room events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-            var roomInvokeCount = 0;
-            var roomNameInvokeCount = 0;
-            var timelineFireCount = 0;
+            let roomInvokeCount = 0;
+            let roomNameInvokeCount = 0;
+            let timelineFireCount = 0;
             client.on("Room", function(room) {
                 roomInvokeCount++;
                 expect(room.roomId).toEqual("!erufh:bar");
@@ -176,35 +190,37 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(roomInvokeCount).toEqual(
-                    1, "Room fired wrong number of times."
+                    1, "Room fired wrong number of times.",
                 );
                 expect(roomNameInvokeCount).toEqual(
-                    1, "Room.name fired wrong number of times."
+                    1, "Room.name fired wrong number of times.",
                 );
                 expect(timelineFireCount).toEqual(
-                    3, "Room.timeline fired the wrong number of times"
+                    3, "Room.timeline fired the wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit RoomState events", function(done) {
+        it("should emit RoomState events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
-            var roomStateEventTypes = [
-                "m.room.member", "m.room.create"
+            const roomStateEventTypes = [
+                "m.room.member", "m.room.create",
             ];
-            var eventsInvokeCount = 0;
-            var membersInvokeCount = 0;
-            var newMemberInvokeCount = 0;
+            let eventsInvokeCount = 0;
+            let membersInvokeCount = 0;
+            let newMemberInvokeCount = 0;
             client.on("RoomState.events", function(event, state) {
                 eventsInvokeCount++;
-                var index = roomStateEventTypes.indexOf(event.getType());
-                expect(index).not.toEqual(
-                    -1, "Unexpected room state event type: " + event.getType()
+                const index = roomStateEventTypes.indexOf(event.getType());
+                expect(index).toNotEqual(
+                    -1, "Unexpected room state event type: " + event.getType(),
                 );
                 if (index >= 0) {
                     roomStateEventTypes.splice(index, 1);
@@ -225,28 +241,30 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(membersInvokeCount).toEqual(
-                    1, "RoomState.members fired wrong number of times"
+                    1, "RoomState.members fired wrong number of times",
                 );
                 expect(newMemberInvokeCount).toEqual(
-                    1, "RoomState.newMember fired wrong number of times"
+                    1, "RoomState.newMember fired wrong number of times",
                 );
                 expect(eventsInvokeCount).toEqual(
-                    2, "RoomState.events fired wrong number of times"
+                    2, "RoomState.events fired wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit RoomMember events", function(done) {
+        it("should emit RoomMember events", function() {
             httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
 
-            var typingInvokeCount = 0;
-            var powerLevelInvokeCount = 0;
-            var nameInvokeCount = 0;
-            var membershipInvokeCount = 0;
+            let typingInvokeCount = 0;
+            let powerLevelInvokeCount = 0;
+            let nameInvokeCount = 0;
+            let membershipInvokeCount = 0;
             client.on("RoomMember.name", function(event, member) {
                 nameInvokeCount++;
             });
@@ -264,40 +282,40 @@ describe("MatrixClient events", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                 expect(typingInvokeCount).toEqual(
-                    1, "RoomMember.typing fired wrong number of times"
+                    1, "RoomMember.typing fired wrong number of times",
                 );
                 expect(powerLevelInvokeCount).toEqual(
-                    0, "RoomMember.powerLevel fired wrong number of times"
+                    0, "RoomMember.powerLevel fired wrong number of times",
                 );
                 expect(nameInvokeCount).toEqual(
-                    0, "RoomMember.name fired wrong number of times"
+                    0, "RoomMember.name fired wrong number of times",
                 );
                 expect(membershipInvokeCount).toEqual(
-                    1, "RoomMember.membership fired wrong number of times"
+                    1, "RoomMember.membership fired wrong number of times",
                 );
-                done();
             });
         });
 
-        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function(done) {
+        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function() {
             httpBackend.when("GET", "/sync").respond(401, { errcode: 'M_UNKNOWN_TOKEN' });
 
-            var sessionLoggedOutCount = 0;
+            let sessionLoggedOutCount = 0;
             client.on("Session.logged_out", function(event, member) {
                 sessionLoggedOutCount++;
             });
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return httpBackend.flushAllExpected().then(function() {
                 expect(sessionLoggedOutCount).toEqual(
-                    1, "Session.logged_out fired wrong number of times"
+                    1, "Session.logged_out fired wrong number of times",
                 );
-                done();
             });
         });
     });
-
 });

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,29 +1,30 @@
 "use strict";
-var q = require("q");
-var sdk = require("../..");
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
-var EventTimeline = sdk.EventTimeline;
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+const EventTimeline = sdk.EventTimeline;
 
-var baseUrl = "http://localhost.or.something";
-var userId = "@alice:localhost";
-var userName = "Alice";
-var accessToken = "aseukfgwef";
-var roomId = "!foo:bar";
-var otherUserId = "@bob:localhost";
+const baseUrl = "http://localhost.or.something";
+const userId = "@alice:localhost";
+const userName = "Alice";
+const accessToken = "aseukfgwef";
+const roomId = "!foo:bar";
+const otherUserId = "@bob:localhost";
 
-var USER_MEMBERSHIP_EVENT = utils.mkMembership({
-    room: roomId, mship: "join", user: userId, name: userName
+const USER_MEMBERSHIP_EVENT = utils.mkMembership({
+    room: roomId, mship: "join", user: userId, name: userName,
 });
 
-var ROOM_NAME_EVENT = utils.mkEvent({
+const ROOM_NAME_EVENT = utils.mkEvent({
     type: "m.room.name", room: roomId, user: otherUserId,
     content: {
-        name: "Old room name"
-    }
+        name: "Old room name",
+    },
 });
 
-var INITIAL_SYNC_DATA = {
+const INITIAL_SYNC_DATA = {
     next_batch: "s_5_3",
     rooms: {
         join: {
@@ -31,33 +32,33 @@ var INITIAL_SYNC_DATA = {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "hello"
-                        })
+                            room: roomId, user: otherUserId, msg: "hello",
+                        }),
                     ],
-                    prev_batch: "f_1_1"
+                    prev_batch: "f_1_1",
                 },
                 state: {
                     events: [
                         ROOM_NAME_EVENT,
                         utils.mkMembership({
                             room: roomId, mship: "join",
-                            user: otherUserId, name: "Bob"
+                            user: otherUserId, name: "Bob",
                         }),
                         USER_MEMBERSHIP_EVENT,
                         utils.mkEvent({
                             type: "m.room.create", room: roomId, user: userId,
                             content: {
-                                creator: userId
-                            }
-                        })
-                    ]
-                }
-            }
-        }
-    }
+                                creator: userId,
+                            },
+                        }),
+                    ],
+                },
+            },
+        },
+    },
 };
 
-var EVENTS = [
+const EVENTS = [
     utils.mkMessage({
         room: roomId, user: userId, msg: "we",
     }),
@@ -81,7 +82,7 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    var deferred = q.defer();
+    const deferred = Promise.defer();
     client.on("sync", function(state) {
         console.log("sync", state);
         if (state != "SYNCING") {
@@ -90,14 +91,15 @@ function startClient(httpBackend, client) {
         deferred.resolve();
     });
 
-    httpBackend.flush();
-    return deferred.promise;
+    return Promise.all([
+        httpBackend.flushAllExpected(),
+        deferred.promise,
+    ]);
 }
 
-
 describe("getEventTimeline support", function() {
-    var httpBackend;
-    var client;
+    let httpBackend;
+    let client;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -109,6 +111,7 @@ describe("getEventTimeline support", function() {
         if (client) {
             client.stopClient();
         }
+        return httpBackend.stop();
     });
 
     it("timeline support must be enabled to work", function(done) {
@@ -118,16 +121,17 @@ describe("getEventTimeline support", function() {
             accessToken: accessToken,
         });
 
-        startClient(httpBackend, client
+        startClient(httpBackend, client,
         ).then(function() {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
-            expect(function() { client.getEventTimeline(timelineSet, "event"); })
-                .toThrow();
-        }).catch(utils.failTest).done(done);
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+            expect(function() {
+                client.getEventTimeline(timelineSet, "event");
+            }).toThrow();
+        }).nodeify(done);
     });
 
-    it("timeline support works when enabled", function(done) {
+    it("timeline support works when enabled", function() {
         client = sdk.createClient({
             baseUrl: baseUrl,
             userId: userId,
@@ -135,15 +139,13 @@ describe("getEventTimeline support", function() {
             timelineSupport: true,
         });
 
-        startClient(httpBackend, client
-        ).then(function() {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
-            expect(function() { client.getEventTimeline(timelineSet, "event"); })
-                .not.toThrow();
-        }).catch(utils.failTest).done(done);
-
-        httpBackend.flush().catch(utils.failTest);
+        return startClient(httpBackend, client).then(() => {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
+            expect(function() {
+                client.getEventTimeline(timelineSet, "event");
+            }).toNotThrow();
+        });
     });
 
 
@@ -155,9 +157,9 @@ describe("getEventTimeline support", function() {
             userId: userId,
             accessToken: accessToken,
         });
-        var room;
+        let room;
 
-        startClient(httpBackend, client
+        startClient(httpBackend, client,
         ).then(function() {
             room = client.getRoom(roomId);
 
@@ -194,18 +196,19 @@ describe("getEventTimeline support", function() {
                 },
             });
 
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]);
+        }).then(function() {
+            expect(room.timeline.length).toEqual(1);
+            expect(room.timeline[0].event).toEqual(EVENTS[1]);
+
             httpBackend.when("GET", "/messages").respond(200, {
                 chunk: [EVENTS[0]],
                 start: "pagin_start",
                 end: "pagin_end",
             });
-
-
-            return httpBackend.flush("/sync", 2);
-        }).then(function() {
-            expect(room.timeline.length).toEqual(1);
-            expect(room.timeline[0].event).toEqual(EVENTS[1]);
-
             httpBackend.flush("/messages", 1);
             return client.scrollback(room);
         }).then(function() {
@@ -213,14 +216,17 @@ describe("getEventTimeline support", function() {
             expect(room.timeline[0].event).toEqual(EVENTS[0]);
             expect(room.timeline[1].event).toEqual(EVENTS[1]);
             expect(room.oldState.paginationToken).toEqual("pagin_end");
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 });
 
-describe("MatrixClient event timelines", function() {
-    var client, httpBackend;
+import expect from 'expect';
 
-    beforeEach(function(done) {
+describe("MatrixClient event timelines", function() {
+    let client = null;
+    let httpBackend = null;
+
+    beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
         httpBackend = new HttpBackend();
         sdk.request(httpBackend.requestFn);
@@ -232,8 +238,7 @@ describe("MatrixClient event timelines", function() {
             timelineSupport: true,
         });
 
-        startClient(httpBackend, client)
-            .catch(utils.failTest).done(done);
+        return startClient(httpBackend, client);
     });
 
     afterEach(function() {
@@ -242,9 +247,9 @@ describe("MatrixClient event timelines", function() {
     });
 
     describe("getEventTimeline", function() {
-        it("should create a new timeline for new events", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should create a new timeline for new events", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/event1%3Abar")
                 .respond(200, function() {
                     return {
@@ -260,24 +265,25 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
-                expect(tl.getEvents().length).toEqual(4);
-                for (var i = 0; i < 4; i++) {
-                    expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
-                    expect(tl.getEvents()[i].sender.name).toEqual(userName);
-                }
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(4);
+                    for (let i = 0; i < 4; i++) {
+                        expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
+                        expect(tl.getEvents()[i].sender.name).toEqual(userName);
+                    }
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
-        it("should return existing timeline for known events", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should return existing timeline for known events", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/sync").respond(200, {
                 next_batch: "s_5_4",
                 rooms: {
@@ -294,22 +300,25 @@ describe("MatrixClient event timelines", function() {
                 },
             });
 
-            httpBackend.flush("/sync").then(function() {
+            return Promise.all([
+                httpBackend.flush("/sync"),
+                utils.syncPromise(client),
+            ]).then(function() {
                 return client.getEventTimeline(timelineSet, EVENTS[0].event_id);
             }).then(function(tl) {
                 expect(tl.getEvents().length).toEqual(2);
                 expect(tl.getEvents()[1].event).toEqual(EVENTS[0]);
                 expect(tl.getEvents()[1].sender.name).toEqual(userName);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS)).toEqual("f_1_1");
-                // expect(tl.getPaginationToken(EventTimeline.FORWARDS)).toEqual("s_5_4");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                    .toEqual("f_1_1");
+                // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                //    .toEqual("s_5_4");
+            });
         });
 
-        it("should update timelines where they overlap a previous /sync", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should update timelines where they overlap a previous /sync", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
             httpBackend.when("GET", "/sync").respond(200, {
                 next_batch: "s_5_4",
                 rooms: {
@@ -339,8 +348,9 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
+            const deferred = Promise.defer();
             client.on("sync", function() {
-                client.getEventTimeline(timelineSet, EVENTS[2].event_id
+                client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                 ).then(function(tl) {
                     expect(tl.getEvents().length).toEqual(4);
                     expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
@@ -350,16 +360,20 @@ describe("MatrixClient event timelines", function() {
                         .toEqual("start_token");
                     // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
                     //    .toEqual("s_5_4");
-                }).catch(utils.failTest).done(done);
+                }).done(() => deferred.resolve(),
+                        (e) => deferred.reject(e));
             });
 
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                deferred.promise,
+            ]);
         });
 
         it("should join timelines where they overlap a previous /context",
-          function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+          function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
 
             // we fetch event 0, then 2, then 3, and finally 1. 1 is returned
             // with context which joins them all up.
@@ -415,44 +429,46 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            var tl0, tl3;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id
-            ).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl0 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl3 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
-            }).then(function(tl) {
-                // we expect it to get merged in with event 2
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
-                expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
-                    .toBe(tl0);
-                expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
-                    .toBe(tl3);
-                expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token3");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            let tl0;
+            let tl3;
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl0 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl3 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
+                }).then(function(tl) {
+                    // we expect it to get merged in with event 2
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
+                        .toBe(tl0);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
+                        .toBe(tl3);
+                    expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token3");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
-        it("should fail gracefully if there is no event field", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should fail gracefully if there is no event field", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
             // we fetch event 0, then 2, then 3, and finally 1. 1 is returned
             // with context which joins them all up.
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/event1")
@@ -466,22 +482,23 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            client.getEventTimeline(timelineSet, "event1"
-            ).then(function(tl) {
-                // could do with a fail()
-                expect(true).toBeFalsy();
-            }).catch(function(e) {
-                expect(String(e)).toMatch(/'event'/);
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1",
+                ).then(function(tl) {
+                    // could do with a fail()
+                    expect(true).toBeFalsy();
+                }, function(e) {
+                    expect(String(e)).toMatch(/'event'/);
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
     });
 
     describe("paginateEventTimeline", function() {
-        it("should allow you to paginate backwards", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should allow you to paginate backwards", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
 
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/" +
                              encodeURIComponent(EVENTS[0].event_id))
@@ -498,7 +515,7 @@ describe("MatrixClient event timelines", function() {
 
             httpBackend.when("GET", "/rooms/!foo%3Abar/messages")
                 .check(function(req) {
-                    var params = req.queryParams;
+                    const params = req.queryParams;
                     expect(params.dir).toEqual("b");
                     expect(params.from).toEqual("start_token0");
                     expect(params.limit).toEqual(30);
@@ -509,30 +526,31 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            var tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(tl, {backwards: true});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token1");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token0");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            let tl;
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(tl, {backwards: true});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token1");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token0");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
 
 
-        it("should allow you to paginate forwards", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should allow you to paginate forwards", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
 
             httpBackend.when("GET", "/rooms/!foo%3Abar/context/" +
                              encodeURIComponent(EVENTS[0].event_id))
@@ -549,7 +567,7 @@ describe("MatrixClient event timelines", function() {
 
             httpBackend.when("GET", "/rooms/!foo%3Abar/messages")
                 .check(function(req) {
-                    var params = req.queryParams;
+                    const params = req.queryParams;
                     expect(params.dir).toEqual("f");
                     expect(params.from).toEqual("end_token0");
                     expect(params.limit).toEqual(20);
@@ -560,31 +578,32 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            var tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(
-                    tl, {backwards: false, limit: 20});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token1");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            let tl;
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(
+                        tl, {backwards: false, limit: 20});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token1");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
         });
     });
 
     describe("event timeline for sent events", function() {
-        var TXN_ID = "txn1";
-        var event = utils.mkMessage({
+        const TXN_ID = "txn1";
+        const event = utils.mkMessage({
             room: roomId, user: userId, msg: "a body",
         });
         event.unsigned = {transaction_id: TXN_ID};
@@ -603,7 +622,7 @@ describe("MatrixClient event timelines", function() {
                         "!foo:bar": {
                             timeline: {
                                 events: [
-                                    event
+                                    event,
                                 ],
                                 prev_batch: "f_1_1",
                             },
@@ -613,56 +632,66 @@ describe("MatrixClient event timelines", function() {
             });
         });
 
-        it("should work when /send returns before /sync", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should work when /send returns before /sync", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
 
-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                // 2 because the initial sync contained an event
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+            return Promise.all([
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    // 2 because the initial sync contained an event
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");
 
-                // now let the sync complete, and check it again
-                return httpBackend.flush("/sync", 1);
-            }).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
-            }).catch(utils.failTest).done(done);
+                    // now let the sync complete, and check it again
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);
+                }),
 
-            httpBackend.flush("/send/m.room.message/" + TXN_ID, 1).catch(utils.failTest);
+                httpBackend.flush("/send/m.room.message/" + TXN_ID, 1),
+            ]);
         });
 
-        it("should work when /send returns after /sync", function(done) {
-            var room = client.getRoom(roomId);
-            var timelineSet = room.getTimelineSets()[0];
+        it("should work when /send returns after /sync", function() {
+            const room = client.getRoom(roomId);
+            const timelineSet = room.getTimelineSets()[0];
 
-            // initiate the send, and set up checks to be done when it completes
-            // - but note that it won't complete until after the /sync does, below.
-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                console.log("sendTextMessage completed");
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (2)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
-            }).catch(utils.failTest).done(done);
+            return Promise.all([
+                // initiate the send, and set up checks to be done when it completes
+                // - but note that it won't complete until after the /sync does, below.
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    console.log("sendTextMessage completed");
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (2)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+                }),
 
-            httpBackend.flush("/sync", 1).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (1)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
+                Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (1)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);
 
-                // now let the send complete.
-                return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
-            }).catch(utils.failTest);
+                    // now let the send complete.
+                    return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
+                }),
+            ]);
         });
     });
 
@@ -671,18 +700,18 @@ describe("MatrixClient event timelines", function() {
         // https://github.com/vector-im/vector-web/issues/1389
 
         // a state event, followed by a redaction thereof
-        var event = utils.mkMembership({
-            room: roomId, mship: "join", user: otherUserId
+        const event = utils.mkMembership({
+            room: roomId, mship: "join", user: otherUserId,
         });
-        var redaction = utils.mkEvent({
+        const redaction = utils.mkEvent({
             type: "m.room.redaction",
             room_id: roomId,
             sender: otherUserId,
-            content: {}
+            content: {},
         });
         redaction.redacts = event.event_id;
 
-        var syncData = {
+        const syncData = {
             next_batch: "batch1",
             rooms: {
                 join: {},
@@ -699,13 +728,16 @@ describe("MatrixClient event timelines", function() {
         };
         httpBackend.when("GET", "/sync").respond(200, syncData);
 
-        httpBackend.flush().then(function() {
-            var room = client.getRoom(roomId);
-            var tl = room.getLiveTimeline();
+        Promise.all([
+            httpBackend.flushAllExpected(),
+            utils.syncPromise(client),
+        ]).then(function() {
+            const room = client.getRoom(roomId);
+            const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(3);
             expect(tl.getEvents()[1].isRedacted()).toBe(true);
 
-            var sync2 = {
+            const sync2 = {
                 next_batch: "batch2",
                 rooms: {
                     join: {},
@@ -715,7 +747,7 @@ describe("MatrixClient event timelines", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomId, user: otherUserId, msg: "world"
+                            room: roomId, user: otherUserId, msg: "world",
                         }),
                     ],
                     limited: true,
@@ -724,11 +756,14 @@ describe("MatrixClient event timelines", function() {
             };
             httpBackend.when("GET", "/sync").respond(200, sync2);
 
-            return httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client),
+            ]);
         }).then(function() {
-            var room = client.getRoom(roomId);
-            var tl = room.getLiveTimeline();
+            const room = client.getRoom(roomId);
+            const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
     });
 });

spec/integ/matrix-client-methods.spec.js

@@ -1,25 +1,31 @@
 "use strict";
-var sdk = require("../..");
-var HttpBackend = require("../mock-request");
-var publicGlobals = require("../../lib/matrix");
-var Room = publicGlobals.Room;
-var MatrixInMemoryStore = publicGlobals.MatrixInMemoryStore;
-var Filter = publicGlobals.Filter;
-var utils = require("../test-utils");
-var MockStorageApi = require("../MockStorageApi");
+import 'source-map-support/register';
+const sdk = require("../..");
+const HttpBackend = require("matrix-mock-request");
+const publicGlobals = require("../../lib/matrix");
+const Room = publicGlobals.Room;
+const MatrixInMemoryStore = publicGlobals.MatrixInMemoryStore;
+const Filter = publicGlobals.Filter;
+const utils = require("../test-utils");
+const MockStorageApi = require("../MockStorageApi");
+
+import expect from 'expect';
 
 describe("MatrixClient", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend, store, sessionStore;
-    var userId = "@alice:localhost";
-    var accessToken = "aseukfgwef";
+    const baseUrl = "http://localhost.or.something";
+    let client = null;
+    let httpBackend = null;
+    let store = null;
+    let sessionStore = null;
+    const userId = "@alice:localhost";
+    const accessToken = "aseukfgwef";
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
         httpBackend = new HttpBackend();
         store = new MatrixInMemoryStore();
 
-        var mockStorage = new MockStorageApi();
+        const mockStorage = new MockStorageApi();
         sessionStore = new sdk.WebStorageSessionStore(mockStorage);
 
         sdk.request(httpBackend.requestFn);
@@ -35,31 +41,35 @@ describe("MatrixClient", function() {
 
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
     });
 
     describe("uploadContent", function() {
-        var buf = new Buffer('hello world');
+        const buf = new Buffer('hello world');
         it("should upload the file", function(done) {
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload"
+                "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                 expect(req.queryParams.filename).toEqual("hi.txt");
-                expect(req.queryParams.access_token).toEqual(accessToken);
+                if (!(req.queryParams.access_token == accessToken ||
+                        req.headers["Authorization"] == "Bearer " + accessToken)) {
+                    expect(true).toBe(false);
+                }
                 expect(req.headers["Content-Type"]).toEqual("text/plain");
                 expect(req.opts.json).toBeFalsy();
                 expect(req.opts.timeout).toBe(undefined);
-            }).respond(200, "content");
+            }).respond(200, "content", true);
 
-            var prom = client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
             });
 
-            expect(prom).toBeDefined();
+            expect(prom).toBeTruthy();
 
-            var uploads = client.getCurrentUploads();
+            const uploads = client.getCurrentUploads();
             expect(uploads.length).toEqual(1);
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
@@ -68,19 +78,19 @@ describe("MatrixClient", function() {
                 // for backwards compatibility, we return the raw JSON
                 expect(response).toEqual("content");
 
-                var uploads = client.getCurrentUploads();
+                const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
 
         it("should parse the response if rawResponse=false", function(done) {
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload"
+                "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
                 expect(req.opts.json).toBeFalsy();
-            }).respond(200, JSON.stringify({ "content_uri": "uri" }));
+            }).respond(200, { "content_uri": "uri" });
 
             client.uploadContent({
                 stream: buf,
@@ -90,22 +100,21 @@ describe("MatrixClient", function() {
                 rawResponse: false,
             }).then(function(response) {
                 expect(response.content_uri).toEqual("uri");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
 
         it("should parse errors into a MatrixError", function(done) {
-            // opts.json is false, so request returns unparsed json.
             httpBackend.when(
-                "POST", "/_matrix/media/v1/upload"
+                "POST", "/_matrix/media/v1/upload",
             ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                 expect(req.opts.json).toBeFalsy();
-            }).respond(400, JSON.stringify({
+            }).respond(400, {
                 "errcode": "M_SNAFU",
                 "error": "broken",
-            }));
+            });
 
             client.uploadContent({
                 stream: buf,
@@ -117,19 +126,19 @@ describe("MatrixClient", function() {
                 expect(error.httpStatus).toEqual(400);
                 expect(error.errcode).toEqual("M_SNAFU");
                 expect(error.message).toEqual("broken");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
 
         it("should return a promise which can be cancelled", function(done) {
-            var prom = client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
             });
 
-            var uploads = client.getCurrentUploads();
+            const uploads = client.getCurrentUploads();
             expect(uploads.length).toEqual(1);
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
@@ -139,23 +148,23 @@ describe("MatrixClient", function() {
             }, function(error) {
                 expect(error).toEqual("aborted");
 
-                var uploads = client.getCurrentUploads();
+                const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
-            var r = client.cancelUpload(prom);
+            const r = client.cancelUpload(prom);
             expect(r).toBe(true);
         });
     });
 
     describe("joinRoom", function() {
         it("should no-op if you've already joined a room", function() {
-            var roomId = "!foo:bar";
-            var room = new Room(roomId);
+            const roomId = "!foo:bar";
+            const room = new Room(roomId, userId);
             room.addLiveEvents([
                 utils.mkMembership({
-                    user: userId, room: roomId, mship: "join", event: true
-                })
+                    user: userId, room: roomId, mship: "join", event: true,
+                }),
             ]);
             store.storeRoom(room);
             client.joinRoom(roomId);
@@ -164,11 +173,11 @@ describe("MatrixClient", function() {
     });
 
     describe("getFilter", function() {
-        var filterId = "f1lt3r1d";
+        const filterId = "f1lt3r1d";
 
         it("should return a filter from the store if allowCached", function(done) {
-            var filter = Filter.fromJson(userId, filterId, {
-                event_format: "client"
+            const filter = Filter.fromJson(userId, filterId, {
+                event_format: "client",
             });
             store.storeFilter(filter);
             client.getFilter(userId, filterId, true).done(function(gotFilter) {
@@ -180,16 +189,16 @@ describe("MatrixClient", function() {
 
         it("should do an HTTP request if !allowCached even if one exists",
         function(done) {
-            var httpFilterDefinition = {
-                event_format: "federation"
+            const httpFilterDefinition = {
+                event_format: "federation",
             };
 
             httpBackend.when(
-                "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId
+                "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
             ).respond(200, httpFilterDefinition);
 
-            var storeFilter = Filter.fromJson(userId, filterId, {
-                event_format: "client"
+            const storeFilter = Filter.fromJson(userId, filterId, {
+                event_format: "client",
             });
             store.storeFilter(storeFilter);
             client.getFilter(userId, filterId, false).done(function(gotFilter) {
@@ -202,17 +211,17 @@ describe("MatrixClient", function() {
 
         it("should do an HTTP request if nothing is in the cache and then store it",
         function(done) {
-            var httpFilterDefinition = {
-                event_format: "federation"
+            const httpFilterDefinition = {
+                event_format: "federation",
             };
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);
 
             httpBackend.when(
-                "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId
+                "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
             ).respond(200, httpFilterDefinition);
             client.getFilter(userId, filterId, true).done(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
-                expect(store.getFilter(userId, filterId)).toBeDefined();
+                expect(store.getFilter(userId, filterId)).toBeTruthy();
                 done();
             });
 
@@ -221,21 +230,21 @@ describe("MatrixClient", function() {
     });
 
     describe("createFilter", function() {
-        var filterId = "f1llllllerid";
+        const filterId = "f1llllllerid";
 
         it("should do an HTTP request and then store the filter", function(done) {
-            expect(store.getFilter(userId, filterId)).toBeNull();
+            expect(store.getFilter(userId, filterId)).toBe(null);
 
-            var filterDefinition = {
-                event_format: "client"
+            const filterDefinition = {
+                event_format: "client",
             };
 
             httpBackend.when(
-                "POST", "/user/" + encodeURIComponent(userId) + "/filter"
+                "POST", "/user/" + encodeURIComponent(userId) + "/filter",
             ).check(function(req) {
                 expect(req.data).toEqual(filterDefinition);
             }).respond(200, {
-                filter_id: filterId
+                filter_id: filterId,
             });
 
             client.createFilter(filterDefinition).done(function(gotFilter) {
@@ -249,8 +258,7 @@ describe("MatrixClient", function() {
     });
 
     describe("searching", function() {
-
-        var response = {
+        const response = {
             search_categories: {
                 room_events: {
                     count: 24,
@@ -263,26 +271,26 @@ describe("MatrixClient", function() {
                                 room_id: "!feuiwhf:localhost",
                                 content: {
                                     body: "a result",
-                                    msgtype: "m.text"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
+                                    msgtype: "m.text",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
         };
 
         it("searchMessageText should perform a /search for room_events", function(done) {
             client.searchMessageText({
-                query: "monkeys"
+                query: "monkeys",
             });
             httpBackend.when("POST", "/search").check(function(req) {
                 expect(req.data).toEqual({
                     search_categories: {
                         room_events: {
-                            search_term: "monkeys"
-                        }
-                    }
+                            search_term: "monkeys",
+                        },
+                    },
                 });
             }).respond(200, response);
 
@@ -294,10 +302,18 @@ describe("MatrixClient", function() {
 
 
     describe("downloadKeys", function() {
+        if (!sdk.CRYPTO_ENABLED) {
+            return;
+        }
+
+        beforeEach(function() {
+            return client.initCrypto();
+        });
+
         it("should do an HTTP request and then store the keys", function(done) {
-            var ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
+            const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
-            var borisKeys = {
+            const borisKeys = {
                 dev1: {
                     algorithms: ["1"],
                     device_id: "dev1",
@@ -306,14 +322,14 @@ describe("MatrixClient", function() {
                         boris: {
                             "ed25519:dev1":
                                 "RAhmbNDq1efK3hCpBzZDsKoGSsrHUxb25NW5/WbEV9R" +
-                                "JVwLdP032mg5QsKt/pBDUGtggBcnk43n3nBWlA88WAw"
-                        }
+                                "JVwLdP032mg5QsKt/pBDUGtggBcnk43n3nBWlA88WAw",
+                        },
                     },
                     unsigned: { "abc": "def" },
                     user_id: "boris",
-                }
+                },
             };
-            var chazKeys = {
+            const chazKeys = {
                 dev2: {
                     algorithms: ["2"],
                     device_id: "dev2",
@@ -322,12 +338,12 @@ describe("MatrixClient", function() {
                         chaz: {
                            "ed25519:dev2":
                                 "FwslH/Q7EYSb7swDJbNB5PSzcbEO1xRRBF1riuijqvL" +
-                                "EkrK9/XVN8jl4h7thGuRITQ01siBQnNmMK9t45QfcCQ"
-                        }
+                                "EkrK9/XVN8jl4h7thGuRITQ01siBQnNmMK9t45QfcCQ",
+                        },
                     },
                     unsigned: { "ghi": "def" },
                     user_id: "chaz",
-                }
+                },
             };
 
             /*
@@ -345,7 +361,10 @@ describe("MatrixClient", function() {
             */
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
-                expect(req.data).toEqual({device_keys: {boris: {}, chaz: {}}});
+                expect(req.data).toEqual({device_keys: {
+                    'boris': {},
+                    'chaz': {},
+                }});
             }).respond(200, {
                 device_keys: {
                     boris: borisKeys,
@@ -367,24 +386,24 @@ describe("MatrixClient", function() {
                     algorithms: ["2"],
                     unsigned: { "ghi": "def" },
                 });
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
 
             httpBackend.flush();
         });
     });
 
     describe("deleteDevice", function() {
-        var auth = {a: 1};
+        const auth = {a: 1};
         it("should pass through an auth dict", function(done) {
             httpBackend.when(
-                "DELETE", "/_matrix/client/unstable/devices/my_device"
+                "DELETE", "/_matrix/client/unstable/devices/my_device",
             ).check(function(req) {
                 expect(req.data).toEqual({auth: auth});
             }).respond(200);
 
             client.deleteDevice(
-                "my_device", auth
-            ).catch(utils.failTest).done(done);
+                "my_device", auth,
+            ).nodeify(done);
 
             httpBackend.flush();
         });
@@ -392,7 +411,7 @@ describe("MatrixClient", function() {
 });
 
 function assertObjectContains(obj, expected) {
-    for (var k in expected) {
+    for (const k in expected) {
         if (expected.hasOwnProperty(k)) {
             expect(obj[k]).toEqual(expected[k]);
         }

spec/integ/matrix-client-opts.spec.js

@@ -1,17 +1,22 @@
 "use strict";
-var sdk = require("../..");
-var MatrixClient = sdk.MatrixClient;
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const MatrixClient = sdk.MatrixClient;
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+
+import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient opts", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend;
-    var userId = "@alice:localhost";
-    var userB = "@bob:localhost";
-    var accessToken = "aseukfgwef";
-    var roomId = "!foo:bar";
-    var syncData = {
+    const baseUrl = "http://localhost.or.something";
+    let client = null;
+    let httpBackend = null;
+    const userId = "@alice:localhost";
+    const userB = "@bob:localhost";
+    const accessToken = "aseukfgwef";
+    const roomId = "!foo:bar";
+    const syncData = {
         next_batch: "s_5_3",
         presence: {},
         rooms: {
@@ -20,36 +25,36 @@ describe("MatrixClient opts", function() {
                     timeline: {
                         events: [
                             utils.mkMessage({
-                                room: roomId, user: userB, msg: "hello"
-                            })
+                                room: roomId, user: userB, msg: "hello",
+                            }),
                         ],
-                        prev_batch: "f_1_1"
+                        prev_batch: "f_1_1",
                     },
                     state: {
                         events: [
                             utils.mkEvent({
                                 type: "m.room.name", room: roomId, user: userB,
                                 content: {
-                                    name: "Old room name"
-                                }
+                                    name: "Old room name",
+                                },
                             }),
                             utils.mkMembership({
-                                room: roomId, mship: "join", user: userB, name: "Bob"
+                                room: roomId, mship: "join", user: userB, name: "Bob",
                             }),
                             utils.mkMembership({
-                                room: roomId, mship: "join", user: userId, name: "Alice"
+                                room: roomId, mship: "join", user: userId, name: "Alice",
                             }),
                             utils.mkEvent({
                                 type: "m.room.create", room: roomId, user: userId,
                                 content: {
-                                    creator: userId
-                                }
-                            })
-                        ]
-                    }
-                }
-            }
-        }
+                                    creator: userId,
+                                },
+                            }),
+                        ],
+                    },
+                },
+            },
+        },
     };
 
     beforeEach(function() {
@@ -59,6 +64,7 @@ describe("MatrixClient opts", function() {
 
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
     });
 
     describe("without opts.store", function() {
@@ -69,7 +75,7 @@ describe("MatrixClient opts", function() {
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
-                scheduler: new sdk.MatrixScheduler()
+                scheduler: new sdk.MatrixScheduler(),
             });
         });
 
@@ -78,9 +84,9 @@ describe("MatrixClient opts", function() {
         });
 
         it("should be able to send messages", function(done) {
-            var eventId = "$flibble:wibble";
+            const eventId = "$flibble:wibble";
             httpBackend.when("PUT", "/txn1").respond(200, {
-                event_id: eventId
+                event_id: eventId,
             });
             client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
                 expect(res.event_id).toEqual(eventId);
@@ -89,33 +95,32 @@ describe("MatrixClient opts", function() {
             httpBackend.flush("/txn1", 1);
         });
 
-        it("should be able to sync / get new events", function(done) {
-            var expectedEventTypes = [ // from /initialSync
+        it("should be able to sync / get new events", async function() {
+            const expectedEventTypes = [ // from /initialSync
                 "m.room.message", "m.room.name", "m.room.member", "m.room.member",
-                "m.room.create"
+                "m.room.create",
             ];
             client.on("event", function(event) {
-                expect(expectedEventTypes.indexOf(event.getType())).not.toEqual(
-                    -1, "Recv unexpected event type: " + event.getType()
+                expect(expectedEventTypes.indexOf(event.getType())).toNotEqual(
+                    -1, "Recv unexpected event type: " + event.getType(),
                 );
                 expectedEventTypes.splice(
-                    expectedEventTypes.indexOf(event.getType()), 1
+                    expectedEventTypes.indexOf(event.getType()), 1,
                 );
             });
             httpBackend.when("GET", "/pushrules").respond(200, {});
             httpBackend.when("POST", "/filter").respond(200, { filter_id: "foo" });
             httpBackend.when("GET", "/sync").respond(200, syncData);
-            client.startClient();
-            httpBackend.flush("/pushrules", 1).then(function() {
-                return httpBackend.flush("/filter", 1);
-            }).then(function() {
-                return httpBackend.flush("/sync", 1);
-            }).done(function() {
-                expect(expectedEventTypes.length).toEqual(
-                    0, "Expected to see event types: " + expectedEventTypes
-                );
-                done();
-            });
+            await client.startClient();
+            await httpBackend.flush("/pushrules", 1);
+            await httpBackend.flush("/filter", 1);
+            await Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]);
+            expect(expectedEventTypes.length).toEqual(
+                0, "Expected to see event types: " + expectedEventTypes,
+            );
         });
     });
 
@@ -127,14 +132,14 @@ describe("MatrixClient opts", function() {
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
-                scheduler: undefined
+                scheduler: undefined,
             });
         });
 
         it("shouldn't retry sending events", function(done) {
             httpBackend.when("PUT", "/txn1").fail(500, {
                 errcode: "M_SOMETHING",
-                error: "Ruh roh"
+                error: "Ruh roh",
             });
             client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
@@ -147,13 +152,13 @@ describe("MatrixClient opts", function() {
 
         it("shouldn't queue events", function(done) {
             httpBackend.when("PUT", "/txn1").respond(200, {
-                event_id: "AAA"
+                event_id: "AAA",
             });
             httpBackend.when("PUT", "/txn2").respond(200, {
-                event_id: "BBB"
+                event_id: "BBB",
             });
-            var sentA = false;
-            var sentB = false;
+            let sentA = false;
+            let sentB = false;
             client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
                 sentA = true;
                 expect(sentB).toBe(true);
@@ -171,7 +176,7 @@ describe("MatrixClient opts", function() {
 
         it("should be able to send messages", function(done) {
             httpBackend.when("PUT", "/txn1").respond(200, {
-                event_id: "foo"
+                event_id: "foo",
             });
             client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
                 expect(res.event_id).toEqual("foo");

spec/integ/matrix-client-retrying.spec.js

@@ -1,17 +1,23 @@
 "use strict";
-var sdk = require("../..");
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
-var EventStatus = sdk.EventStatus;
+import 'source-map-support/register';
+import Promise from 'bluebird';
+
+const sdk = require("../..");
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+const EventStatus = sdk.EventStatus;
+
+import expect from 'expect';
 
 describe("MatrixClient retrying", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend;
-    var scheduler;
-    var userId = "@alice:localhost";
-    var accessToken = "aseukfgwef";
-    var roomId = "!room:here";
-    var room;
+    const baseUrl = "http://localhost.or.something";
+    let client = null;
+    let httpBackend = null;
+    let scheduler;
+    const userId = "@alice:localhost";
+    const accessToken = "aseukfgwef";
+    const roomId = "!room:here";
+    let room;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -30,6 +36,7 @@ describe("MatrixClient retrying", function() {
 
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
     });
 
     xit("should retry according to MatrixScheduler.retryFn", function() {
@@ -48,22 +55,25 @@ describe("MatrixClient retrying", function() {
 
     });
 
-    it("should mark events as EventStatus.CANCELLED when cancelled", function(done) {
-
+    it("should mark events as EventStatus.CANCELLED when cancelled", function() {
         // send a couple of events; the second will be queued
-        var ev1, ev2;
-        client.sendMessage(roomId, "m1").then(function(ev) {
-            expect(ev).toEqual(ev1);
-        });
-        client.sendMessage(roomId, "m2").then(function(ev) {
-            expect(ev).toEqual(ev2);
+        const p1 = client.sendMessage(roomId, "m1").then(function(ev) {
+            // we expect the first message to fail
+            throw new Error('Message 1 unexpectedly sent successfully');
+        }, (e) => {
+            // this is expected
         });
 
+        // XXX: it turns out that the promise returned by this message
+        // never gets resolved.
+        // https://github.com/matrix-org/matrix-js-sdk/issues/496
+        client.sendMessage(roomId, "m2");
+
         // both events should be in the timeline at this point
-        var tl = room.getLiveTimeline().getEvents();
+        const tl = room.getLiveTimeline().getEvents();
         expect(tl.length).toEqual(2);
-        ev1 = tl[0];
-        ev2 = tl[1];
+        const ev1 = tl[0];
+        const ev2 = tl[1];
 
         expect(ev1.status).toEqual(EventStatus.SENDING);
         expect(ev2.status).toEqual(EventStatus.SENDING);
@@ -79,11 +89,19 @@ describe("MatrixClient retrying", function() {
             expect(tl.length).toEqual(1);
 
             // shouldn't be able to cancel the first message yet
-            expect(function() { client.cancelPendingEvent(ev1); })
-                .toThrow();
+            expect(function() {
+                client.cancelPendingEvent(ev1);
+            }).toThrow();
         }).respond(400); // fail the first message
 
-        httpBackend.flush().then(function() {
+        // wait for the localecho of ev1 to be updated
+        const p3 = new Promise((resolve, reject) => {
+            room.on("Room.localEchoUpdated", (ev0) => {
+                if(ev0 === ev1) {
+                    resolve();
+                }
+            });
+        }).then(function() {
             expect(ev1.status).toEqual(EventStatus.NOT_SENT);
             expect(tl.length).toEqual(1);
 
@@ -91,7 +109,13 @@ describe("MatrixClient retrying", function() {
             client.cancelPendingEvent(ev1);
             expect(ev1.status).toEqual(EventStatus.CANCELLED);
             expect(tl.length).toEqual(0);
-        }).catch(utils.failTest).done(done);
+        });
+
+        return Promise.all([
+            p1,
+            p3,
+            httpBackend.flushAllExpected(),
+        ]);
     });
 
     describe("resending", function() {

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,28 +1,33 @@
 "use strict";
-var sdk = require("../..");
-var EventStatus = sdk.EventStatus;
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const EventStatus = sdk.EventStatus;
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+
+import Promise from 'bluebird';
+import expect from 'expect';
 
 describe("MatrixClient room timelines", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend;
-    var userId = "@alice:localhost";
-    var userName = "Alice";
-    var accessToken = "aseukfgwef";
-    var roomId = "!foo:bar";
-    var otherUserId = "@bob:localhost";
-    var USER_MEMBERSHIP_EVENT = utils.mkMembership({
-        room: roomId, mship: "join", user: userId, name: userName
+    const baseUrl = "http://localhost.or.something";
+    let client = null;
+    let httpBackend = null;
+    const userId = "@alice:localhost";
+    const userName = "Alice";
+    const accessToken = "aseukfgwef";
+    const roomId = "!foo:bar";
+    const otherUserId = "@bob:localhost";
+    const USER_MEMBERSHIP_EVENT = utils.mkMembership({
+        room: roomId, mship: "join", user: userId, name: userName,
     });
-    var ROOM_NAME_EVENT = utils.mkEvent({
+    const ROOM_NAME_EVENT = utils.mkEvent({
         type: "m.room.name", room: roomId, user: otherUserId,
         content: {
-            name: "Old room name"
-        }
+            name: "Old room name",
+        },
     });
-    var NEXT_SYNC_DATA;
-    var SYNC_DATA = {
+    let NEXT_SYNC_DATA;
+    const SYNC_DATA = {
         next_batch: "s_5_3",
         rooms: {
             join: {
@@ -30,30 +35,30 @@ describe("MatrixClient room timelines", function() {
                     timeline: {
                         events: [
                             utils.mkMessage({
-                                room: roomId, user: otherUserId, msg: "hello"
-                            })
+                                room: roomId, user: otherUserId, msg: "hello",
+                            }),
                         ],
-                        prev_batch: "f_1_1"
+                        prev_batch: "f_1_1",
                     },
                     state: {
                         events: [
                             ROOM_NAME_EVENT,
                             utils.mkMembership({
                                 room: roomId, mship: "join",
-                                user: otherUserId, name: "Bob"
+                                user: otherUserId, name: "Bob",
                             }),
                             USER_MEMBERSHIP_EVENT,
                             utils.mkEvent({
                                 type: "m.room.create", room: roomId, user: userId,
                                 content: {
-                                    creator: userId
-                                }
-                            })
-                        ]
-                    }
-                }
-            }
-        }
+                                    creator: userId,
+                                },
+                            }),
+                        ],
+                    },
+                },
+            },
+        },
     };
 
     function setNextSyncData(events) {
@@ -67,11 +72,11 @@ describe("MatrixClient room timelines", function() {
                     "!foo:bar": {
                         timeline: { events: [] },
                         state: { events: [] },
-                        ephemeral: { events: [] }
-                    }
+                        ephemeral: { events: [] },
+                    },
                 },
-                leave: {}
-            }
+                leave: {},
+            },
         };
         events.forEach(function(e) {
             if (e.room_id !== roomId) {
@@ -81,7 +86,7 @@ describe("MatrixClient room timelines", function() {
                 if (e.__prev_event === undefined) {
                     throw new Error(
                         "setNextSyncData needs the prev state set to '__prev_event' " +
-                        "for " + e.type
+                        "for " + e.type,
                     );
                 }
                 if (e.__prev_event !== null) {
@@ -90,11 +95,9 @@ describe("MatrixClient room timelines", function() {
                 }
                 // push the current
                 NEXT_SYNC_DATA.rooms.join[roomId].timeline.events.push(e);
-            }
-            else if (["m.typing", "m.receipt"].indexOf(e.type) !== -1) {
+            } else if (["m.typing", "m.receipt"].indexOf(e.type) !== -1) {
                 NEXT_SYNC_DATA.rooms.join[roomId].ephemeral.events.push(e);
-            }
-            else {
+            } else {
                 NEXT_SYNC_DATA.rooms.join[roomId].timeline.events.push(e);
             }
         });
@@ -121,21 +124,23 @@ describe("MatrixClient room timelines", function() {
         client.startClient();
         httpBackend.flush("/pushrules").then(function() {
             return httpBackend.flush("/filter");
-        }).done(done);
+        }).nodeify(done);
     });
 
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
         client.stopClient();
+        return httpBackend.stop();
     });
 
     describe("local echo events", function() {
-
         it("should be added immediately after calling MatrixClient.sendEvent " +
         "with EventStatus.SENDING and the right event.sender", function(done) {
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
                 client.sendTextMessage(roomId, "I am a fish", "txn1");
@@ -144,7 +149,7 @@ describe("MatrixClient room timelines", function() {
                 // check status
                 expect(room.timeline[1].status).toEqual(EventStatus.SENDING);
                 // check member
-                var member = room.timeline[1].sender;
+                const member = room.timeline[1].sender;
                 expect(member.userId).toEqual(userId);
                 expect(member.name).toEqual(userName);
 
@@ -157,21 +162,23 @@ describe("MatrixClient room timelines", function() {
 
         it("should be updated correctly when the send request finishes " +
         "BEFORE the event comes down the event stream", function(done) {
-            var eventId = "$foo:bar";
+            const eventId = "$foo:bar";
             httpBackend.when("PUT", "/txn1").respond(200, {
-                event_id: eventId
+                event_id: eventId,
             });
 
-            var ev = utils.mkMessage({
-                body: "I am a fish", user: userId, room: roomId
+            const ev = utils.mkMessage({
+                body: "I am a fish", user: userId, room: roomId,
             });
             ev.event_id = eventId;
             ev.unsigned = {transaction_id: "txn1"};
             setNextSyncData([ev]);
 
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
                 client.sendTextMessage(roomId, "I am a fish", "txn1").done(
                 function() {
                     expect(room.timeline[1].getId()).toEqual(eventId);
@@ -187,22 +194,24 @@ describe("MatrixClient room timelines", function() {
 
         it("should be updated correctly when the send request finishes " +
         "AFTER the event comes down the event stream", function(done) {
-            var eventId = "$foo:bar";
+            const eventId = "$foo:bar";
             httpBackend.when("PUT", "/txn1").respond(200, {
-                event_id: eventId
+                event_id: eventId,
             });
 
-            var ev = utils.mkMessage({
-                body: "I am a fish", user: userId, room: roomId
+            const ev = utils.mkMessage({
+                body: "I am a fish", user: userId, room: roomId,
             });
             ev.event_id = eventId;
             ev.unsigned = {transaction_id: "txn1"};
             setNextSyncData([ev]);
 
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
-                var promise = client.sendTextMessage(roomId, "I am a fish", "txn1");
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
+                const promise = client.sendTextMessage(roomId, "I am a fish", "txn1");
                 httpBackend.flush("/sync", 1).done(function() {
                     expect(room.timeline.length).toEqual(2);
                     httpBackend.flush("/txn1", 1);
@@ -212,15 +221,14 @@ describe("MatrixClient room timelines", function() {
                         done();
                     });
                 });
-
             });
             httpBackend.flush("/sync", 1);
         });
     });
 
     describe("paginated events", function() {
-        var sbEvents;
-        var sbEndTok = "pagin_end";
+        let sbEvents;
+        const sbEndTok = "pagin_end";
 
         beforeEach(function() {
             sbEvents = [];
@@ -228,7 +236,7 @@ describe("MatrixClient room timelines", function() {
                 return {
                     chunk: sbEvents,
                     start: "pagin_start",
-                    end: sbEndTok
+                    end: sbEndTok,
                 };
             });
         });
@@ -236,18 +244,23 @@ describe("MatrixClient room timelines", function() {
         it("should set Room.oldState.paginationToken to null at the start" +
         " of the timeline.", function(done) {
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
                 client.scrollback(room).done(function() {
                     expect(room.timeline.length).toEqual(1);
-                    expect(room.oldState.paginationToken).toBeNull();
-                    done();
+                    expect(room.oldState.paginationToken).toBe(null);
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -263,55 +276,60 @@ describe("MatrixClient room timelines", function() {
             // <Bob> hello
 
             // make an m.room.member event for alice's join
-            var joinMshipEvent = utils.mkMembership({
+            const joinMshipEvent = utils.mkMembership({
                 mship: "join", user: userId, room: roomId, name: "Old Alice",
-                url: null
+                url: null,
             });
 
             // make an m.room.member event with prev_content for alice's nick
             // change
-            var oldMshipEvent = utils.mkMembership({
+            const oldMshipEvent = utils.mkMembership({
                 mship: "join", user: userId, room: roomId, name: userName,
-                url: "mxc://some/url"
+                url: "mxc://some/url",
             });
             oldMshipEvent.prev_content = {
                 displayname: "Old Alice",
                 avatar_url: null,
-                membership: "join"
+                membership: "join",
             };
 
             // set the list of events to return on scrollback (/messages)
             // N.B. synapse returns /messages in reverse chronological order
             sbEvents = [
                 utils.mkMessage({
-                    user: userId, room: roomId, msg: "I'm alice"
+                    user: userId, room: roomId, msg: "I'm alice",
                 }),
                 oldMshipEvent,
                 utils.mkMessage({
-                    user: userId, room: roomId, msg: "I'm old alice"
+                    user: userId, room: roomId, msg: "I'm old alice",
                 }),
                 joinMshipEvent,
             ];
 
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
                 // sync response
                 expect(room.timeline.length).toEqual(1);
 
                 client.scrollback(room).done(function() {
                     expect(room.timeline.length).toEqual(5);
-                    var joinMsg = room.timeline[0];
+                    const joinMsg = room.timeline[0];
                     expect(joinMsg.sender.name).toEqual("Old Alice");
-                    var oldMsg = room.timeline[1];
+                    const oldMsg = room.timeline[1];
                     expect(oldMsg.sender.name).toEqual("Old Alice");
-                    var newMsg = room.timeline[3];
+                    const newMsg = room.timeline[3];
                     expect(newMsg.sender.name).toEqual(userName);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -320,27 +338,32 @@ describe("MatrixClient room timelines", function() {
             // set the list of events to return on scrollback
             sbEvents = [
                 utils.mkMessage({
-                    user: userId, room: roomId, msg: "I am new"
+                    user: userId, room: roomId, msg: "I am new",
                 }),
                 utils.mkMessage({
-                    user: userId, room: roomId, msg: "I am old"
-                })
+                    user: userId, room: roomId, msg: "I am old",
+                }),
             ];
 
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
                 client.scrollback(room).done(function() {
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[0].event).toEqual(sbEvents[1]);
                     expect(room.timeline[1].event).toEqual(sbEvents[0]);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
             });
             httpBackend.flush("/sync", 1);
         });
@@ -349,22 +372,26 @@ describe("MatrixClient room timelines", function() {
             // set the list of events to return on scrollback
             sbEvents = [
                 utils.mkMessage({
-                    user: userId, room: roomId, msg: "I am new"
-                })
+                    user: userId, room: roomId, msg: "I am new",
+                }),
             ];
 
             client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
-                expect(room.oldState.paginationToken).toBeDefined();
+                if (state !== "PREPARED") {
+                    return;
+                }
+                const room = client.getRoom(roomId);
+                expect(room.oldState.paginationToken).toBeTruthy();
 
                 client.scrollback(room, 1).done(function() {
                     expect(room.oldState.paginationToken).toEqual(sbEndTok);
                 });
 
-                httpBackend.flush("/sync", 1);
                 httpBackend.flush("/messages", 1).done(function() {
-                    done();
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                 });
             });
             httpBackend.flush("/sync", 1);
@@ -372,18 +399,20 @@ describe("MatrixClient room timelines", function() {
     });
 
     describe("new events", function() {
-        it("should be added to the right place in the timeline", function(done) {
-            var eventData = [
+        it("should be added to the right place in the timeline", function() {
+            const eventData = [
+                utils.mkMessage({user: userId, room: roomId}),
                 utils.mkMessage({user: userId, room: roomId}),
-                utils.mkMessage({user: userId, room: roomId})
             ];
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
 
-                var index = 0;
+                let index = 0;
                 client.on("Room.timeline", function(event, rm, toStart) {
                     expect(toStart).toBe(false);
                     expect(rm).toEqual(room);
@@ -392,172 +421,194 @@ describe("MatrixClient room timelines", function() {
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(index).toEqual(2);
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[2].event).toEqual(
-                        eventData[1]
+                        eventData[1],
                     );
                     expect(room.timeline[1].event).toEqual(
-                        eventData[0]
+                        eventData[0],
                     );
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right event.sender values", function(done) {
-            var eventData = [
+        it("should set the right event.sender values", function() {
+            const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
                 utils.mkMembership({
-                    user: userId, room: roomId, mship: "join", name: "New Name"
+                    user: userId, room: roomId, mship: "join", name: "New Name",
                 }),
-                utils.mkMessage({user: userId, room: roomId})
+                utils.mkMessage({user: userId, room: roomId}),
             ];
             eventData[1].__prev_event = USER_MEMBERSHIP_EVENT;
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
-                    var preNameEvent = room.timeline[room.timeline.length - 3];
-                    var postNameEvent = room.timeline[room.timeline.length - 1];
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
+                    const preNameEvent = room.timeline[room.timeline.length - 3];
+                    const postNameEvent = room.timeline[room.timeline.length - 1];
                     expect(preNameEvent.sender.name).toEqual(userName);
                     expect(postNameEvent.sender.name).toEqual("New Name");
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right room.name", function(done) {
-            var secondRoomNameEvent = utils.mkEvent({
+        it("should set the right room.name", function() {
+            const secondRoomNameEvent = utils.mkEvent({
                 user: userId, room: roomId, type: "m.room.name", content: {
-                    name: "Room 2"
-                }
+                    name: "Room 2",
+                },
             });
             secondRoomNameEvent.__prev_event = ROOM_NAME_EVENT;
             setNextSyncData([secondRoomNameEvent]);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
-                var nameEmitCount = 0;
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
+                let nameEmitCount = 0;
                 client.on("Room.name", function(rm) {
                     nameEmitCount += 1;
                 });
 
-                httpBackend.flush("/sync", 1).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(nameEmitCount).toEqual(1);
                     expect(room.name).toEqual("Room 2");
                     // do another round
-                    var thirdRoomNameEvent = utils.mkEvent({
+                    const thirdRoomNameEvent = utils.mkEvent({
                         user: userId, room: roomId, type: "m.room.name", content: {
-                            name: "Room 3"
-                        }
+                            name: "Room 3",
+                        },
                     });
                     thirdRoomNameEvent.__prev_event = secondRoomNameEvent;
                     setNextSyncData([thirdRoomNameEvent]);
                     httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-                    httpBackend.flush("/sync", 1).done(function() {
-                        expect(nameEmitCount).toEqual(2);
-                        expect(room.name).toEqual("Room 3");
-                        done();
-                    });
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    expect(nameEmitCount).toEqual(2);
+                    expect(room.name).toEqual("Room 3");
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should set the right room members", function(done) {
-            var userC = "@cee:bar";
-            var userD = "@dee:bar";
-            var eventData = [
+        it("should set the right room members", function() {
+            const userC = "@cee:bar";
+            const userD = "@dee:bar";
+            const eventData = [
                 utils.mkMembership({
-                    user: userC, room: roomId, mship: "join", name: "C"
+                    user: userC, room: roomId, mship: "join", name: "C",
                 }),
                 utils.mkMembership({
-                    user: userC, room: roomId, mship: "invite", skey: userD
-                })
+                    user: userC, room: roomId, mship: "invite", skey: userD,
+                }),
             ];
             eventData[0].__prev_event = null;
             eventData[1].__prev_event = null;
             setNextSyncData(eventData);
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(room.currentState.getMembers().length).toEqual(4);
                     expect(room.currentState.getMember(userC).name).toEqual("C");
                     expect(room.currentState.getMember(userC).membership).toEqual(
-                        "join"
+                        "join",
                     );
                     expect(room.currentState.getMember(userD).name).toEqual(userD);
                     expect(room.currentState.getMember(userD).membership).toEqual(
-                        "invite"
+                        "invite",
                     );
-                }).catch(utils.failTest).done(done);
+                });
             });
-            httpBackend.flush("/sync", 1);
         });
     });
 
     describe("gappy sync", function() {
-        it("should copy the last known state to the new timeline", function(done) {
-            var eventData = [
+        it("should copy the last known state to the new timeline", function() {
+            const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.timeline[0].event).toEqual(eventData[0]);
                     expect(room.currentState.getMembers().length).toEqual(2);
                     expect(room.currentState.getMember(userId).name).toEqual(userName);
                     expect(room.currentState.getMember(userId).membership).toEqual(
-                        "join"
+                        "join",
                     );
                     expect(room.currentState.getMember(otherUserId).name).toEqual("Bob");
                     expect(room.currentState.getMember(otherUserId).membership).toEqual(
-                        "join"
+                        "join",
                     );
-                    done();
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
 
-        it("should emit a 'Room.timelineReset' event", function(done) {
-            var eventData = [
+        it("should emit a 'Room.timelineReset' event", function() {
+            const eventData = [
                 utils.mkMessage({user: userId, room: roomId}),
             ];
             setNextSyncData(eventData);
             NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;
 
-            client.on("sync", function(state) {
-                if (state !== "PREPARED") { return; }
-                var room = client.getRoom(roomId);
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
+                const room = client.getRoom(roomId);
 
-                var emitCount = 0;
+                let emitCount = 0;
                 client.on("Room.timelineReset", function(emitRoom) {
                     expect(emitRoom).toEqual(room);
                     emitCount++;
                 });
 
                 httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                     expect(emitCount).toEqual(1);
-                    done();
                 });
             });
-            httpBackend.flush("/sync", 1);
         });
     });
 });

spec/integ/matrix-client-syncing.spec.js

@@ -1,21 +1,26 @@
 "use strict";
-var sdk = require("../..");
-var HttpBackend = require("../mock-request");
-var utils = require("../test-utils");
-var MatrixEvent = sdk.MatrixEvent;
-var EventTimeline = sdk.EventTimeline;
+import 'source-map-support/register';
+const sdk = require("../..");
+const HttpBackend = require("matrix-mock-request");
+const utils = require("../test-utils");
+const MatrixEvent = sdk.MatrixEvent;
+const EventTimeline = sdk.EventTimeline;
+
+import expect from 'expect';
+import Promise from 'bluebird';
 
 describe("MatrixClient syncing", function() {
-    var baseUrl = "http://localhost.or.something";
-    var client, httpBackend;
-    var selfUserId = "@alice:localhost";
-    var selfAccessToken = "aseukfgwef";
-    var otherUserId = "@bob:localhost";
-    var userA = "@alice:bar";
-    var userB = "@bob:bar";
-    var userC = "@claire:bar";
-    var roomOne = "!foo:localhost";
-    var roomTwo = "!bar:localhost";
+    const baseUrl = "http://localhost.or.something";
+    let client = null;
+    let httpBackend = null;
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+    const otherUserId = "@bob:localhost";
+    const userA = "@alice:bar";
+    const userB = "@bob:bar";
+    const userC = "@claire:bar";
+    const roomOne = "!foo:localhost";
+    const roomTwo = "!bar:localhost";
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -24,7 +29,7 @@ describe("MatrixClient syncing", function() {
         client = sdk.createClient({
             baseUrl: baseUrl,
             userId: selfUserId,
-            accessToken: selfAccessToken
+            accessToken: selfAccessToken,
         });
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
@@ -33,13 +38,14 @@ describe("MatrixClient syncing", function() {
     afterEach(function() {
         httpBackend.verifyNoOutstandingExpectation();
         client.stopClient();
+        return httpBackend.stop();
     });
 
     describe("startClient", function() {
-        var syncData = {
+        const syncData = {
             next_batch: "batch_token",
             rooms: {},
-            presence: {}
+            presence: {},
         };
 
         it("should /sync after /pushrules and /filter.", function(done) {
@@ -47,7 +53,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 done();
             });
         });
@@ -61,24 +67,23 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                 done();
             });
         });
     });
 
     describe("resolving invites to profile info", function() {
-
-        var syncData = {
+        const syncData = {
             next_batch: "s_5_3",
             presence: {
-                events: []
+                events: [],
             },
             rooms: {
                 join: {
 
-                }
-            }
+                },
+            },
         };
 
         beforeEach(function() {
@@ -87,98 +92,103 @@ describe("MatrixClient syncing", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomOne, user: otherUserId, msg: "hello"
-                        })
-                    ]
+                            room: roomOne, user: otherUserId, msg: "hello",
+                        }),
+                    ],
                 },
                 state: {
                     events: [
                         utils.mkMembership({
-                            room: roomOne, mship: "join", user: otherUserId
+                            room: roomOne, mship: "join", user: otherUserId,
                         }),
                         utils.mkMembership({
-                            room: roomOne, mship: "join", user: selfUserId
+                            room: roomOne, mship: "join", user: selfUserId,
                         }),
                         utils.mkEvent({
                             type: "m.room.create", room: roomOne, user: selfUserId,
                             content: {
-                                creator: selfUserId
-                            }
-                        })
-                    ]
-                }
+                                creator: selfUserId,
+                            },
+                        }),
+                    ],
+                },
             };
         });
 
-        it("should resolve incoming invites from /sync", function(done) {
+        it("should resolve incoming invites from /sync", function() {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
-                    room: roomOne, mship: "invite", user: userC
-                })
+                    room: roomOne, mship: "invite", user: userC,
+                }),
             );
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/profile/" + encodeURIComponent(userC)).respond(
                 200, {
                     avatar_url: "mxc://flibble/wibble",
-                    displayname: "The Boss"
-                }
+                    displayname: "The Boss",
+                },
             );
 
             client.startClient({
-                resolveInvitesToProfiles: true
+                resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
-                var member = client.getRoom(roomOne).getMember(userC);
+
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual("The Boss");
                 expect(
-                    member.getAvatarUrl("home.server.url", null, null, null, false)
-                ).toBeDefined();
-                done();
+                    member.getAvatarUrl("home.server.url", null, null, null, false),
+                ).toBeTruthy();
             });
         });
 
-        it("should use cached values from m.presence wherever possible", function(done) {
+        it("should use cached values from m.presence wherever possible", function() {
             syncData.presence.events = [
                 utils.mkPresence({
-                    user: userC, presence: "online", name: "The Ghost"
+                    user: userC, presence: "online", name: "The Ghost",
                 }),
             ];
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
-                    room: roomOne, mship: "invite", user: userC
-                })
+                    room: roomOne, mship: "invite", user: userC,
+                }),
             );
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient({
-                resolveInvitesToProfiles: true
+                resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
-                var member = client.getRoom(roomOne).getMember(userC);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual("The Ghost");
-                done();
             });
         });
 
-        it("should result in events on the room member firing", function(done) {
+        it("should result in events on the room member firing", function() {
             syncData.presence.events = [
                 utils.mkPresence({
-                    user: userC, presence: "online", name: "The Ghost"
-                })
+                    user: userC, presence: "online", name: "The Ghost",
+                }),
             ];
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
-                    room: roomOne, mship: "invite", user: userC
-                })
+                    room: roomOne, mship: "invite", user: userC,
+                }),
             );
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            var latestFiredName = null;
+            let latestFiredName = null;
             client.on("RoomMember.name", function(event, m) {
                 if (m.userId === userC && m.roomId === roomOne) {
                     latestFiredName = m.name;
@@ -186,141 +196,147 @@ describe("MatrixClient syncing", function() {
             });
 
             client.startClient({
-                resolveInvitesToProfiles: true
+                resolveInvitesToProfiles: true,
             });
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 expect(latestFiredName).toEqual("The Ghost");
-                done();
             });
         });
 
-        it("should no-op if resolveInvitesToProfiles is not set", function(done) {
+        it("should no-op if resolveInvitesToProfiles is not set", function() {
             syncData.rooms.join[roomOne].state.events.push(
                 utils.mkMembership({
-                    room: roomOne, mship: "invite", user: userC
-                })
+                    room: roomOne, mship: "invite", user: userC,
+                }),
             );
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
-                var member = client.getRoom(roomOne).getMember(userC);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const member = client.getRoom(roomOne).getMember(userC);
                 expect(member.name).toEqual(userC);
                 expect(
-                    member.getAvatarUrl("home.server.url", null, null, null, false)
-                ).toBeNull();
-                done();
+                    member.getAvatarUrl("home.server.url", null, null, null, false),
+                ).toBe(null);
             });
         });
     });
 
     describe("users", function() {
-        var syncData = {
+        const syncData = {
             next_batch: "nb",
             presence: {
                 events: [
                     utils.mkPresence({
-                        user: userA, presence: "online"
+                        user: userA, presence: "online",
                     }),
                     utils.mkPresence({
-                        user: userB, presence: "unavailable"
-                    })
-                ]
-            }
+                        user: userB, presence: "unavailable",
+                    }),
+                ],
+            },
         };
 
         it("should create users for presence events from /sync",
-        function(done) {
+        function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                 expect(client.getUser(userA).presence).toEqual("online");
                 expect(client.getUser(userB).presence).toEqual("unavailable");
-                done();
             });
         });
     });
 
     describe("room state", function() {
-        var msgText = "some text here";
-        var otherDisplayName = "Bob Smith";
+        const msgText = "some text here";
+        const otherDisplayName = "Bob Smith";
 
-        var syncData = {
+        const syncData = {
             rooms: {
                 join: {
 
-                }
-            }
+                },
+            },
         };
         syncData.rooms.join[roomOne] = {
             timeline: {
                 events: [
                     utils.mkMessage({
-                        room: roomOne, user: otherUserId, msg: "hello"
-                    })
-                ]
+                        room: roomOne, user: otherUserId, msg: "hello",
+                    }),
+                ],
             },
             state: {
                 events: [
                     utils.mkEvent({
                         type: "m.room.name", room: roomOne, user: otherUserId,
                         content: {
-                            name: "Old room name"
-                        }
+                            name: "Old room name",
+                        },
                     }),
                     utils.mkMembership({
-                        room: roomOne, mship: "join", user: otherUserId
+                        room: roomOne, mship: "join", user: otherUserId,
                     }),
                     utils.mkMembership({
-                        room: roomOne, mship: "join", user: selfUserId
+                        room: roomOne, mship: "join", user: selfUserId,
                     }),
                     utils.mkEvent({
                         type: "m.room.create", room: roomOne, user: selfUserId,
                         content: {
-                            creator: selfUserId
-                        }
-                    })
-                ]
-            }
+                            creator: selfUserId,
+                        },
+                    }),
+                ],
+            },
         };
         syncData.rooms.join[roomTwo] = {
             timeline: {
                 events: [
                     utils.mkMessage({
-                        room: roomTwo, user: otherUserId, msg: "hiii"
-                    })
-                ]
+                        room: roomTwo, user: otherUserId, msg: "hiii",
+                    }),
+                ],
             },
             state: {
                 events: [
                     utils.mkMembership({
                         room: roomTwo, mship: "join", user: otherUserId,
-                        name: otherDisplayName
+                        name: otherDisplayName,
                     }),
                     utils.mkMembership({
-                        room: roomTwo, mship: "join", user: selfUserId
+                        room: roomTwo, mship: "join", user: selfUserId,
                     }),
                     utils.mkEvent({
                         type: "m.room.create", room: roomTwo, user: selfUserId,
                         content: {
-                            creator: selfUserId
-                        }
-                    })
-                ]
-            }
+                            creator: selfUserId,
+                        },
+                    }),
+                ],
+            },
         };
 
-        var nextSyncData = {
+        const nextSyncData = {
             rooms: {
                 join: {
 
-                }
-            }
+                },
+            },
         };
 
         nextSyncData.rooms.join[roomOne] = {
@@ -328,89 +344,125 @@ describe("MatrixClient syncing", function() {
                 events: [
                     utils.mkEvent({
                         type: "m.room.name", room: roomOne, user: selfUserId,
-                        content: { name: "A new room name" }
-                    })
-                ]
-            }
+                        content: { name: "A new room name" },
+                    }),
+                ],
+            },
         };
 
         nextSyncData.rooms.join[roomTwo] = {
             timeline: {
                 events: [
                     utils.mkMessage({
-                        room: roomTwo, user: otherUserId, msg: msgText
-                    })
-                ]
+                        room: roomTwo, user: otherUserId, msg: msgText,
+                    }),
+                ],
             },
             ephemeral: {
                 events: [
                     utils.mkEvent({
                         type: "m.typing", room: roomTwo,
-                        content: { user_ids: [otherUserId] }
-                    })
-                ]
-            }
+                        content: { user_ids: [otherUserId] },
+                    }),
+                ],
+            },
         };
 
-        it("should continually recalculate the right room name.", function(done) {
+        it("should continually recalculate the right room name.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
-                var room = client.getRoom(roomOne);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
                 // should have clobbered the name to the one from /events
                 expect(room.name).toEqual(
-                    nextSyncData.rooms.join[roomOne].state.events[0].content.name
+                    nextSyncData.rooms.join[roomOne].state.events[0].content.name,
                 );
-                done();
             });
         });
 
-        it("should store the right events in the timeline.", function(done) {
+        it("should store the right events in the timeline.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
-                var room = client.getRoom(roomTwo);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomTwo);
                 // should have added the message from /events
                 expect(room.timeline.length).toEqual(2);
                 expect(room.timeline[1].getContent().body).toEqual(msgText);
-                done();
             });
         });
 
-        it("should set the right room name.", function(done) {
+        it("should set the right room name.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
-            httpBackend.flush().done(function() {
-                var room = client.getRoom(roomTwo);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomTwo);
                 // should use the display name of the other person.
                 expect(room.name).toEqual(otherDisplayName);
-                done();
             });
         });
 
-        it("should set the right user's typing flag.", function(done) {
+        it("should set the right user's typing flag.", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
             httpBackend.when("GET", "/sync").respond(200, nextSyncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
-                var room = client.getRoom(roomTwo);
-                var member = room.getMember(otherUserId);
-                expect(member).toBeDefined();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomTwo);
+                let member = room.getMember(otherUserId);
+                expect(member).toBeTruthy();
                 expect(member.typing).toEqual(true);
                 member = room.getMember(selfUserId);
-                expect(member).toBeDefined();
+                expect(member).toBeTruthy();
                 expect(member.typing).toEqual(false);
-                done();
+            });
+        });
+
+        // XXX: This test asserts that the js-sdk obeys the spec and treats state
+        // events that arrive in the incremental sync as if they preceeded the
+        // timeline events, however this breaks peeking, so it's disabled
+        // (see sync.js)
+        xit("should correctly interpret state in incremental sync.", function() {
+            httpBackend.when("GET", "/sync").respond(200, syncData);
+            httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+            client.startClient();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
+                const stateAtStart = room.getLiveTimeline().getState(
+                    EventTimeline.BACKWARDS,
+                );
+                const startRoomNameEvent = stateAtStart.getStateEvents('m.room.name', '');
+                expect(startRoomNameEvent.getContent().name).toEqual('Old room name');
+
+                const stateAtEnd = room.getLiveTimeline().getState(
+                    EventTimeline.FORWARDS,
+                );
+                const endRoomNameEvent = stateAtEnd.getStateEvents('m.room.name', '');
+                expect(endRoomNameEvent.getContent().name).toEqual('A new room name');
             });
         });
 
@@ -425,7 +477,7 @@ describe("MatrixClient syncing", function() {
 
     describe("timeline", function() {
         beforeEach(function() {
-            var syncData = {
+            const syncData = {
                 next_batch: "batch_token",
                 rooms: {
                     join: {},
@@ -435,7 +487,7 @@ describe("MatrixClient syncing", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomOne, user: otherUserId, msg: "hello"
+                            room: roomOne, user: otherUserId, msg: "hello",
                         }),
                     ],
                     prev_batch: "pagTok",
@@ -445,11 +497,14 @@ describe("MatrixClient syncing", function() {
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
-            httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]);
         });
 
-        it("should set the back-pagination token on new rooms", function(done) {
-            var syncData = {
+        it("should set the back-pagination token on new rooms", function() {
+            const syncData = {
                 next_batch: "batch_token",
                 rooms: {
                     join: {},
@@ -459,7 +514,7 @@ describe("MatrixClient syncing", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomTwo, user: otherUserId, msg: "roomtwo"
+                            room: roomTwo, user: otherUserId, msg: "roomtwo",
                         }),
                     ],
                     prev_batch: "roomtwotok",
@@ -468,17 +523,20 @@ describe("MatrixClient syncing", function() {
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            httpBackend.flush().then(function() {
-                var room = client.getRoom(roomTwo);
-                var tok = room.getLiveTimeline()
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const room = client.getRoom(roomTwo);
+                expect(room).toExist();
+                const tok = room.getLiveTimeline()
                     .getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("roomtwotok");
-                done();
-            }).catch(utils.failTest).done();
+            });
         });
 
-        it("should set the back-pagination token on gappy syncs", function(done) {
-            var syncData = {
+        it("should set the back-pagination token on gappy syncs", function() {
+            const syncData = {
                 next_batch: "batch_token",
                 rooms: {
                     join: {},
@@ -488,7 +546,7 @@ describe("MatrixClient syncing", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomOne, user: otherUserId, msg: "world"
+                            room: roomOne, user: otherUserId, msg: "world",
                         }),
                     ],
                     limited: true,
@@ -497,104 +555,108 @@ describe("MatrixClient syncing", function() {
             };
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            var resetCallCount = 0;
+            let resetCallCount = 0;
             // the token should be set *before* timelineReset is emitted
             client.on("Room.timelineReset", function(room) {
                 resetCallCount++;
 
-                var tl = room.getLiveTimeline();
+                const tl = room.getLiveTimeline();
                 expect(tl.getEvents().length).toEqual(0);
-                var tok = tl.getPaginationToken(EventTimeline.BACKWARDS);
+                const tok = tl.getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("newerTok");
             });
 
-            httpBackend.flush().then(function() {
-                var room = client.getRoom(roomOne);
-                var tl = room.getLiveTimeline();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
+                const tl = room.getLiveTimeline();
                 expect(tl.getEvents().length).toEqual(1);
                 expect(resetCallCount).toEqual(1);
-                done();
-            }).catch(utils.failTest).done();
+            });
         });
     });
 
     describe("receipts", function() {
-        var syncData = {
+        const syncData = {
             rooms: {
                 join: {
 
-                }
-            }
+                },
+            },
         };
         syncData.rooms.join[roomOne] = {
             timeline: {
                 events: [
                     utils.mkMessage({
-                        room: roomOne, user: otherUserId, msg: "hello"
+                        room: roomOne, user: otherUserId, msg: "hello",
                     }),
                     utils.mkMessage({
-                        room: roomOne, user: otherUserId, msg: "world"
-                    })
-                ]
+                        room: roomOne, user: otherUserId, msg: "world",
+                    }),
+                ],
             },
             state: {
                 events: [
                     utils.mkEvent({
                         type: "m.room.name", room: roomOne, user: otherUserId,
                         content: {
-                            name: "Old room name"
-                        }
+                            name: "Old room name",
+                        },
                     }),
                     utils.mkMembership({
-                        room: roomOne, mship: "join", user: otherUserId
+                        room: roomOne, mship: "join", user: otherUserId,
                     }),
                     utils.mkMembership({
-                        room: roomOne, mship: "join", user: selfUserId
+                        room: roomOne, mship: "join", user: selfUserId,
                     }),
                     utils.mkEvent({
                         type: "m.room.create", room: roomOne, user: selfUserId,
                         content: {
-                            creator: selfUserId
-                        }
-                    })
-                ]
-            }
+                            creator: selfUserId,
+                        },
+                    }),
+                ],
+            },
         };
 
         beforeEach(function() {
             syncData.rooms.join[roomOne].ephemeral = {
-                events: []
+                events: [],
             };
         });
 
-        it("should sync receipts from /sync.", function(done) {
-            var ackEvent = syncData.rooms.join[roomOne].timeline.events[0];
-            var receipt = {};
+        it("should sync receipts from /sync.", function() {
+            const ackEvent = syncData.rooms.join[roomOne].timeline.events[0];
+            const receipt = {};
             receipt[ackEvent.event_id] = {
-                "m.read": {}
+                "m.read": {},
             };
             receipt[ackEvent.event_id]["m.read"][userC] = {
-                ts: 176592842636
+                ts: 176592842636,
             };
             syncData.rooms.join[roomOne].ephemeral.events = [{
                 content: receipt,
                 room_id: roomOne,
-                type: "m.receipt"
+                type: "m.receipt",
             }];
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
             client.startClient();
 
-            httpBackend.flush().done(function() {
-                var room = client.getRoom(roomOne);
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
                 expect(room.getReceiptsForEvent(new MatrixEvent(ackEvent))).toEqual([{
                     type: "m.read",
                     userId: userC,
                     data: {
-                        ts: 176592842636
-                    }
+                        ts: 176592842636,
+                    },
                 }]);
-                done();
             });
         });
     });
@@ -614,7 +676,7 @@ describe("MatrixClient syncing", function() {
         beforeEach(function(done) {
             client.startClient();
 
-            httpBackend.flush().then(function() {
+            httpBackend.flushAllExpected().then(function() {
                 // the /sync call from syncLeftRooms ends up in the request
                 // queue behind the call from the running client; add a response
                 // to flush the client's one out.
@@ -624,33 +686,38 @@ describe("MatrixClient syncing", function() {
             });
         });
 
-        it("should create and use an appropriate filter", function(done) {
+        it("should create and use an appropriate filter", function() {
             httpBackend.when("POST", "/filter").check(function(req) {
                 expect(req.data).toEqual({
                     room: { timeline: {limit: 1},
                             include_leave: true }});
             }).respond(200, { filter_id: "another_id" });
 
+            const defer = Promise.defer();
+
             httpBackend.when("GET", "/sync").check(function(req) {
                 expect(req.queryParams.filter).toEqual("another_id");
-                done();
+                defer.resolve();
             }).respond(200, {});
 
             client.syncLeftRooms();
 
             // first flush the filter request; this will make syncLeftRooms
             // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                // flush the syncs
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+            return Promise.all([
+                httpBackend.flush("/filter").then(function() {
+                    // flush the syncs
+                    return httpBackend.flushAllExpected();
+                }),
+                defer.promise,
+            ]);
         });
 
-        it("should set the back-pagination token on left rooms", function(done) {
-            var syncData = {
+        it("should set the back-pagination token on left rooms", function() {
+            const syncData = {
                 next_batch: "batch_token",
                 rooms: {
-                    leave: {}
+                    leave: {},
                 },
             };
 
@@ -658,7 +725,7 @@ describe("MatrixClient syncing", function() {
                 timeline: {
                     events: [
                         utils.mkMessage({
-                            room: roomTwo, user: otherUserId, msg: "hello"
+                            room: roomTwo, user: otherUserId, msg: "hello",
                         }),
                     ],
                     prev_batch: "pagTok",
@@ -666,25 +733,36 @@ describe("MatrixClient syncing", function() {
             };
 
             httpBackend.when("POST", "/filter").respond(200, {
-                filter_id: "another_id"
+                filter_id: "another_id",
             });
 
             httpBackend.when("GET", "/sync").respond(200, syncData);
 
-            client.syncLeftRooms().then(function() {
-                var room = client.getRoom(roomTwo);
-                var tok = room.getLiveTimeline().getPaginationToken(
-                    EventTimeline.BACKWARDS);
+            return Promise.all([
+                client.syncLeftRooms().then(function() {
+                    const room = client.getRoom(roomTwo);
+                    const tok = room.getLiveTimeline().getPaginationToken(
+                        EventTimeline.BACKWARDS);
 
-                expect(tok).toEqual("pagTok");
-                done();
-            }).catch(utils.failTest).done();
+                    expect(tok).toEqual("pagTok");
+                }),
 
-            // first flush the filter request; this will make syncLeftRooms
-            // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+                // first flush the filter request; this will make syncLeftRooms
+                // make its /sync call
+                httpBackend.flush("/filter").then(function() {
+                    return httpBackend.flushAllExpected();
+                }),
+            ]);
         });
     });
+
+    /**
+     * waits for the MatrixClient to emit one or more 'sync' events.
+     *
+     * @param {Number?} numSyncs number of syncs to wait for
+     * @returns {Promise} promise which resolves after the sync events have happened
+     */
+    function awaitSyncEvent(numSyncs) {
+        return utils.syncPromise(client, numSyncs);
+    }
 });

spec/mock-request.js

@@ -1,274 +0,0 @@
-"use strict";
-var q = require("q");
-
-/**
- * Construct a mock HTTP backend, heavily inspired by Angular.js.
- * @constructor
- */
-function HttpBackend() {
-    this.requests = [];
-    this.expectedRequests = [];
-    var self = this;
-    // the request function dependency that the SDK needs.
-    this.requestFn = function(opts, callback) {
-        var req = new Request(opts, callback);
-        console.log("HTTP backend received request: %s", req);
-        self.requests.push(req);
-
-        var abort = function() {
-            var idx = self.requests.indexOf(req);
-            if (idx >= 0) {
-                console.log("Aborting HTTP request: %s %s", opts.method,
-                            opts.uri);
-                self.requests.splice(idx, 1);
-                req.callback("aborted");
-            }
-        };
-
-        return {
-            abort: abort
-        };
-    };
-}
-HttpBackend.prototype = {
-    /**
-     * Respond to all of the requests (flush the queue).
-     * @param {string} path The path to flush (optional) default: all.
-     * @param {integer} numToFlush The number of things to flush (optional), default: all.
-     * @return {Promise} resolved when there is nothing left to flush.
-     */
-    flush: function(path, numToFlush) {
-        var defer = q.defer();
-        var self = this;
-        var flushed = 0;
-        var triedWaiting = false;
-        console.log(
-            "HTTP backend flushing... (path=%s  numToFlush=%s)", path, numToFlush
-        );
-        var tryFlush = function() {
-            // if there's more real requests and more expected requests, flush 'em.
-            console.log(
-                "  trying to flush queue => reqs=%s expected=%s [%s]",
-                self.requests.length, self.expectedRequests.length, path
-            );
-            if (self._takeFromQueue(path)) {
-                // try again on the next tick.
-                console.log("  flushed. Trying for more. [%s]", path);
-                flushed += 1;
-                if (numToFlush && flushed === numToFlush) {
-                    console.log("  [%s] Flushed assigned amount: %s", path, numToFlush);
-                    defer.resolve();
-                }
-                else {
-                    setTimeout(tryFlush, 0);
-                }
-            }
-            else if (flushed === 0 && !triedWaiting) {
-                // we may not have made the request yet, wait a generous amount of
-                // time before giving up.
-                setTimeout(tryFlush, 5);
-                triedWaiting = true;
-            }
-            else {
-                console.log("  no more flushes. [%s]", path);
-                defer.resolve();
-            }
-        };
-
-        setTimeout(tryFlush, 0);
-
-        return defer.promise;
-    },
-
-    /**
-     * Attempts to resolve requests/expected requests.
-     * @param {string} path The path to flush (optional) default: all.
-     * @return {boolean} true if something was resolved.
-     */
-    _takeFromQueue: function(path) {
-        var req = null;
-        var i, j;
-        var matchingReq, expectedReq, testResponse = null;
-        for (i = 0; i < this.requests.length; i++) {
-            req = this.requests[i];
-            for (j = 0; j < this.expectedRequests.length; j++) {
-                expectedReq = this.expectedRequests[j];
-                if (path && path !== expectedReq.path) { continue; }
-                if (expectedReq.method === req.method &&
-                        req.path.indexOf(expectedReq.path) !== -1) {
-                    if (!expectedReq.data || (JSON.stringify(expectedReq.data) ===
-                            JSON.stringify(req.data))) {
-                        matchingReq = expectedReq;
-                        this.expectedRequests.splice(j, 1);
-                        break;
-                    }
-                }
-            }
-
-            if (matchingReq) {
-                // remove from request queue
-                this.requests.splice(i, 1);
-                i--;
-
-                for (j = 0; j < matchingReq.checks.length; j++) {
-                    matchingReq.checks[j](req);
-                }
-                testResponse = matchingReq.response;
-                console.log("    responding to %s", matchingReq.path);
-                var body = testResponse.body;
-                if (Object.prototype.toString.call(body) == "[object Function]") {
-                    body = body(req.path, req.data);
-                }
-                req.callback(
-                    testResponse.err, testResponse.response, body
-                );
-                matchingReq = null;
-            }
-        }
-        if (testResponse) {  // flushed something
-            return true;
-        }
-        return false;
-    },
-
-    /**
-     * Makes sure that the SDK hasn't sent any more requests to the backend.
-     */
-    verifyNoOutstandingRequests: function() {
-        var firstOutstandingReq = this.requests[0] || {};
-        expect(this.requests.length).toEqual(0,
-            "Expected no more HTTP requests but received request to " +
-            firstOutstandingReq.path
-        );
-    },
-
-    /**
-     * Makes sure that the test doesn't have any unresolved requests.
-     */
-    verifyNoOutstandingExpectation: function() {
-        var firstOutstandingExpectation = this.expectedRequests[0] || {};
-        expect(this.expectedRequests.length).toEqual(0,
-            "Expected to see HTTP request for " + firstOutstandingExpectation.path
-        );
-    },
-
-    /**
-     * Create an expected request.
-     * @param {string} method The HTTP method
-     * @param {string} path The path (which can be partial)
-     * @param {Object} data The expected data.
-     * @return {Request} An expected request.
-     */
-    when: function(method, path, data) {
-        var pendingReq = new ExpectedRequest(method, path, data);
-        this.expectedRequests.push(pendingReq);
-        return pendingReq;
-    }
-};
-
-/**
- * Represents the expectation of a request.
- *
- * <p>Includes the conditions to be matched against, the checks to be made,
- * and the response to be returned.
- *
- * @constructor
- * @param {string} method
- * @param {string} path
- * @param {object?} data
- */
-function ExpectedRequest(method, path, data) {
-    this.method = method;
-    this.path = path;
-    this.data = data;
-    this.response = null;
-    this.checks = [];
-}
-
-ExpectedRequest.prototype = {
-    /**
-     * Execute a check when this request has been satisfied.
-     * @param {Function} fn The function to execute.
-     * @return {Request} for chaining calls.
-     */
-    check: function(fn) {
-        this.checks.push(fn);
-        return this;
-    },
-
-    /**
-     * Respond with the given data when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Object|Function} data The HTTP JSON body. If this is a function,
-     * it will be invoked when the JSON body is required (which should be returned).
-     */
-    respond: function(code, data) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {}
-            },
-            body: data,
-            err: null
-        };
-    },
-
-    /**
-     * Fail with an Error when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Error} err The error to throw (e.g. Network Error)
-     */
-    fail: function(code, err) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {}
-            },
-            body: null,
-            err: err
-        };
-    }
-};
-
-/**
- * Represents a request made by the app.
- *
- * @constructor
- * @param {object} opts opts passed to request()
- * @param {function} callback
- */
-function Request(opts, callback) {
-    this.opts = opts;
-    this.callback = callback;
-
-    Object.defineProperty(this, 'method', {
-        get: function() { return opts.method; }
-    });
-
-    Object.defineProperty(this, 'path', {
-        get: function() { return opts.uri; }
-    });
-
-    Object.defineProperty(this, 'data', {
-        get: function() { return opts.body; }
-    });
-
-    Object.defineProperty(this, 'queryParams', {
-        get: function() { return opts.qs; }
-    });
-
-    Object.defineProperty(this, 'headers', {
-        get: function() { return opts.headers || {}; }
-    });
-}
-
-Request.prototype = {
-    toString: function() {
-        return this.method + " " + this.path;
-    },
-};
-
-/**
- * The HttpBackend class.
- */
-module.exports = HttpBackend;

spec/olm-loader.js

@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// try to load the olm library.
+
+try {
+    global.Olm = require('olm');
+    console.log('loaded libolm');
+} catch (e) {
+    console.warn("unable to run crypto tests: libolm not available");
+}

spec/test-utils.js

@@ -1,14 +1,53 @@
 "use strict";
-var sdk = require("..");
-var MatrixEvent = sdk.MatrixEvent;
+import expect from 'expect';
+import Promise from 'bluebird';
+
+// load olm before the sdk if possible
+import './olm-loader';
+
+import sdk from '..';
+const MatrixEvent = sdk.MatrixEvent;
+
+/**
+ * Return a promise that is resolved when the client next emits a
+ * SYNCING event.
+ * @param {Object} client The client
+ * @param {Number=} count Number of syncs to wait for (default 1)
+ * @return {Promise} Resolves once the client has emitted a SYNCING event
+ */
+module.exports.syncPromise = function(client, count) {
+    if (count === undefined) {
+        count = 1;
+    }
+    if (count <= 0) {
+        return Promise.resolve();
+    }
+
+    const p = new Promise((resolve, reject) => {
+        const cb = (state) => {
+            console.log(`${Date.now()} syncPromise(${count}): ${state}`);
+            if (state == 'SYNCING') {
+                resolve();
+            } else {
+                client.once('sync', cb);
+            }
+        };
+        client.once('sync', cb);
+    });
+
+    return p.then(() => {
+        return module.exports.syncPromise(client, count-1);
+    });
+};
 
 /**
  * Perform common actions before each test case, e.g. printing the test case
  * name to stdout.
- * @param {TestCase} testCase The test case that is about to be run.
+ * @param {Mocha.Context} context  The test context
  */
-module.exports.beforeEach = function(testCase) {
-    var desc = testCase.suite.description + " : " + testCase.description;
+module.exports.beforeEach = function(context) {
+    const desc = context.currentTest.fullTitle();
+
     console.log(desc);
     console.log(new Array(1 + desc.length).join("="));
 };
@@ -20,21 +59,20 @@ module.exports.beforeEach = function(testCase) {
  * @return {Object} An instantiated object with spied methods/properties.
  */
 module.exports.mock = function(constr, name) {
-    // By Tim Buschtöns
+    // Based on
     // http://eclipsesource.com/blogs/2014/03/27/mocks-in-jasmine-tests/
-    var HelperConstr = new Function(); // jshint ignore:line
+    const HelperConstr = new Function(); // jshint ignore:line
     HelperConstr.prototype = constr.prototype;
-    var result = new HelperConstr();
-    result.jasmineToString = function() {
+    const result = new HelperConstr();
+    result.toString = function() {
         return "mock" + (name ? " of " + name : "");
     };
-    for (var key in constr.prototype) { // eslint-disable-line guard-for-in
+    for (const key in constr.prototype) { // eslint-disable-line guard-for-in
         try {
             if (constr.prototype[key] instanceof Function) {
-                result[key] = jasmine.createSpy((name || "mock") + '.' + key);
+                result[key] = expect.createSpy();
             }
-        }
-        catch (ex) {
+        } catch (ex) {
             // Direct access to some non-function fields of DOM prototypes may
             // cause exceptions.
             // Overwriting will not work either in that case.
@@ -58,17 +96,16 @@ module.exports.mkEvent = function(opts) {
     if (!opts.type || !opts.content) {
         throw new Error("Missing .type or .content =>" + JSON.stringify(opts));
     }
-    var event = {
+    const event = {
         type: opts.type,
         room_id: opts.room,
         sender: opts.sender || opts.user, // opts.user for backwards-compat
         content: opts.content,
-        event_id: "$" + Math.random() + "-" + Math.random()
+        event_id: "$" + Math.random() + "-" + Math.random(),
     };
     if (opts.skey !== undefined) {
         event.state_key = opts.skey;
-    }
-    else if (["m.room.name", "m.room.topic", "m.room.create", "m.room.join_rules",
+    } else if (["m.room.name", "m.room.topic", "m.room.create", "m.room.join_rules",
          "m.room.power_levels", "m.room.topic",
          "com.example.state"].indexOf(opts.type) !== -1) {
         event.state_key = "";
@@ -85,7 +122,7 @@ module.exports.mkPresence = function(opts) {
     if (!opts.user) {
         throw new Error("Missing user");
     }
-    var event = {
+    const event = {
         event_id: "$" + Math.random() + "-" + Math.random(),
         type: "m.presence",
         sender: opts.sender || opts.user, // opts.user for backwards-compat
@@ -93,8 +130,8 @@ module.exports.mkPresence = function(opts) {
             avatar_url: opts.url,
             displayname: opts.name,
             last_active_ago: opts.ago,
-            presence: opts.presence || "offline"
-        }
+            presence: opts.presence || "offline",
+        },
     };
     return opts.event ? new MatrixEvent(event) : event;
 };
@@ -121,10 +158,14 @@ module.exports.mkMembership = function(opts) {
         throw new Error("Missing .mship => " + JSON.stringify(opts));
     }
     opts.content = {
-        membership: opts.mship
+        membership: opts.mship,
     };
-    if (opts.name) { opts.content.displayname = opts.name; }
-    if (opts.url) { opts.content.avatar_url = opts.url; }
+    if (opts.name) {
+        opts.content.displayname = opts.name;
+    }
+    if (opts.url) {
+        opts.content.avatar_url = opts.url;
+    }
     return module.exports.mkEvent(opts);
 };
 
@@ -147,41 +188,12 @@ module.exports.mkMessage = function(opts) {
     }
     opts.content = {
         msgtype: "m.text",
-        body: opts.msg
+        body: opts.msg,
     };
     return module.exports.mkEvent(opts);
 };
 
 
-/**
- * make the test fail, with the given exception
- *
- * <p>This is useful for use with integration tests which use asyncronous
- * methods: it can be added as a 'catch' handler in a promise chain.
- *
- * @param {Error} err   exception to be reported
- *
- * @deprecated
- * It turns out there are easier ways of doing this. Just use nodeify():
- *
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).nodeify(done);
- * });
- *
- * @example
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).catch(utils.failTest).done(done);
- * });
- */
-module.exports.failTest = function(err) {
-    expect(true).toBe(false, "Testfunc threw: " + err.stack);
-};
-
-
 /**
  * A mock implementation of webstorage
  *
@@ -191,6 +203,12 @@ module.exports.MockStorageApi = function() {
     this.data = {};
 };
 module.exports.MockStorageApi.prototype = {
+    get length() {
+        return Object.keys(this.data).length;
+    },
+    key: function(i) {
+        return Object.keys(this.data)[i];
+    },
     setItem: function(k, v) {
         this.data[k] = v;
     },
@@ -199,5 +217,27 @@ module.exports.MockStorageApi.prototype = {
     },
     removeItem: function(k) {
         delete this.data[k];
-    }
+    },
+};
+
+
+/**
+ * If an event is being decrypted, wait for it to finish being decrypted.
+ *
+ * @param {MatrixEvent} event
+ * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
+ */
+module.exports.awaitDecryption = function(event) {
+    if (!event.isBeingDecrypted()) {
+        return Promise.resolve(event);
+    }
+
+    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+
+    return new Promise((resolve, reject) => {
+        event.once('Event.decrypted', (ev) => {
+            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            resolve(ev);
+        });
+    });
 };

spec/unit/content-repo.spec.js

@@ -1,9 +1,12 @@
 "use strict";
-var ContentRepo = require("../../lib/content-repo");
-var testUtils = require("../test-utils");
+import 'source-map-support/register';
+const ContentRepo = require("../../lib/content-repo");
+const testUtils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("ContentRepo", function() {
-    var baseUrl = "https://my.home.server";
+    const baseUrl = "https://my.home.server";
 
     beforeEach(function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -11,24 +14,24 @@ describe("ContentRepo", function() {
 
     describe("getHttpUriForMxc", function() {
         it("should do nothing to HTTP URLs when allowing direct links", function() {
-            var httpUrl = "http://example.com/image.jpeg";
+            const httpUrl = "http://example.com/image.jpeg";
             expect(
                 ContentRepo.getHttpUriForMxc(
-                    baseUrl, httpUrl, undefined, undefined, undefined, true
-                )
+                    baseUrl, httpUrl, undefined, undefined, undefined, true,
+                ),
             ).toEqual(httpUrl);
         });
 
         it("should return the empty string HTTP URLs by default", function() {
-            var httpUrl = "http://example.com/image.jpeg";
+            const httpUrl = "http://example.com/image.jpeg";
             expect(ContentRepo.getHttpUriForMxc(baseUrl, httpUrl)).toEqual("");
         });
 
         it("should return a download URL if no width/height/resize are specified",
         function() {
-            var mxcUri = "mxc://server.name/resourceid";
+            const mxcUri = "mxc://server.name/resourceid";
             expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/v1/download/server.name/resourceid"
+                baseUrl + "/_matrix/media/v1/download/server.name/resourceid",
             );
         });
 
@@ -38,27 +41,27 @@ describe("ContentRepo", function() {
 
         it("should return a thumbnail URL if a width/height/resize is specified",
         function() {
-            var mxcUri = "mxc://server.name/resourceid";
+            const mxcUri = "mxc://server.name/resourceid";
             expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
                 baseUrl + "/_matrix/media/v1/thumbnail/server.name/resourceid" +
-                "?width=32&height=64&method=crop"
+                "?width=32&height=64&method=crop",
             );
         });
 
         it("should put fragments from mxc:// URIs after any query parameters",
         function() {
-            var mxcUri = "mxc://server.name/resourceid#automade";
+            const mxcUri = "mxc://server.name/resourceid#automade";
             expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
                 baseUrl + "/_matrix/media/v1/thumbnail/server.name/resourceid" +
-                "?width=32#automade"
+                "?width=32#automade",
             );
         });
 
         it("should put fragments from mxc:// URIs at the end of the HTTP URI",
         function() {
-            var mxcUri = "mxc://server.name/resourceid#automade";
+            const mxcUri = "mxc://server.name/resourceid#automade";
             expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/v1/download/server.name/resourceid#automade"
+                baseUrl + "/_matrix/media/v1/download/server.name/resourceid#automade",
             );
         });
     });
@@ -71,21 +74,21 @@ describe("ContentRepo", function() {
         it("should set w/h by default to 96", function() {
             expect(ContentRepo.getIdenticonUri(baseUrl, "foobar")).toEqual(
                 baseUrl + "/_matrix/media/v1/identicon/foobar" +
-                "?width=96&height=96"
+                "?width=96&height=96",
             );
         });
 
         it("should be able to set custom w/h", function() {
             expect(ContentRepo.getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
                 baseUrl + "/_matrix/media/v1/identicon/foobar" +
-                "?width=32&height=64"
+                "?width=32&height=64",
             );
         });
 
         it("should URL encode the identicon string", function() {
             expect(ContentRepo.getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
                 baseUrl + "/_matrix/media/v1/identicon/foo%23bar" +
-                "?width=32&height=64"
+                "?width=32&height=64",
             );
         });
     });

spec/unit/crypto.spec.js

@@ -1,14 +1,23 @@
 
 "use strict";
-var Crypto = require("../../lib/crypto");
-var sdk = require("../..");
+import 'source-map-support/register';
+import Crypto from '../../lib/crypto';
+import expect from 'expect';
+
+const sdk = require("../..");
+
+const Olm = global.Olm;
 
 describe("Crypto", function() {
     if (!sdk.CRYPTO_ENABLED) {
         return;
     }
 
+    beforeEach(function(done) {
+        Olm.init().then(done);
+    });
+
     it("Crypto exposes the correct olm library version", function() {
-        expect(Crypto.getOlmVersion()[0]).toEqual(2);
+        expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 });

spec/unit/crypto/DeviceList.spec.js

@@ -0,0 +1,160 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import DeviceList from '../../../lib/crypto/DeviceList';
+import MockStorageApi from '../../MockStorageApi';
+import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
+import testUtils from '../../test-utils';
+import utils from '../../../lib/utils';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+const signedDeviceList = {
+    "failures": {},
+    "device_keys": {
+        "@test1:sw1v.org": {
+            "HGKAWHRVJQ": {
+                "signatures": {
+                    "@test1:sw1v.org": {
+                        "ed25519:HGKAWHRVJQ":
+                            "8PB450fxKDn5s8IiRZ2N2t6MiueQYVRLHFEzqIi1eLdxx1w" +
+                            "XEPC1/1Uz9T4gwnKlMVAKkhB5hXQA/3kjaeLABw",
+                    },
+                },
+                "user_id": "@test1:sw1v.org",
+                "keys": {
+                    "ed25519:HGKAWHRVJQ":
+                        "0gI/T6C+mn1pjtvnnW2yB2l1IIBb/5ULlBXi/LXFSEQ",
+                    "curve25519:HGKAWHRVJQ":
+                        "mbIZED1dBsgIgkgzxDpxKkJmsr4hiWlGzQTvUnQe3RY",
+                },
+                "algorithms": [
+                    "m.olm.v1.curve25519-aes-sha2",
+                    "m.megolm.v1.aes-sha2",
+                ],
+                "device_id": "HGKAWHRVJQ",
+                "unsigned": {},
+            },
+        },
+    },
+};
+
+describe('DeviceList', function() {
+    let downloadSpy;
+    let sessionStore;
+    let cryptoStore;
+    let deviceLists = [];
+
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        deviceLists = [];
+
+        downloadSpy = expect.createSpy();
+        const mockStorage = new MockStorageApi();
+        sessionStore = new WebStorageSessionStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore();
+    });
+
+    afterEach(function() {
+        for (const dl of deviceLists) {
+            dl.stop();
+        }
+    });
+
+    function createTestDeviceList() {
+        const baseApis = {
+            downloadKeysForUsers: downloadSpy,
+        };
+        const mockOlm = {
+            verifySignature: function(key, message, signature) {},
+        };
+        const dl = new DeviceList(baseApis, cryptoStore, sessionStore, mockOlm);
+        deviceLists.push(dl);
+        return dl;
+    }
+
+    it("should successfully download and store device keys", function() {
+        const dl = createTestDeviceList();
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+
+        const queryDefer1 = Promise.defer();
+        downloadSpy.andReturn(queryDefer1.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+        queryDefer1.resolve(utils.deepCopy(signedDeviceList));
+
+        return prom1.then(() => {
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+        });
+    });
+
+    it("should have an outdated devicelist on an invalidation while an " +
+       "update is in progress", function() {
+        const dl = createTestDeviceList();
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+
+        const queryDefer1 = Promise.defer();
+        downloadSpy.andReturn(queryDefer1.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+        downloadSpy.reset();
+
+        // outdated notif arrives while the request is in flight.
+        const queryDefer2 = Promise.defer();
+        downloadSpy.andReturn(queryDefer2.promise);
+
+        dl.invalidateUserDeviceList('@test1:sw1v.org');
+        dl.refreshOutdatedDeviceLists();
+
+        dl.saveIfDirty().then(() => {
+            // the first request completes
+            queryDefer1.resolve({
+                device_keys: {
+                    '@test1:sw1v.org': {},
+                },
+            });
+            return prom1;
+        }).then(() => {
+            // uh-oh; user restarts before second request completes. The new instance
+            // should know we never got a complete device list.
+            console.log("Creating new devicelist to simulate app reload");
+            downloadSpy.reset();
+            const dl2 = createTestDeviceList();
+            const queryDefer3 = Promise.defer();
+            downloadSpy.andReturn(queryDefer3.promise);
+
+            const prom3 = dl2.refreshOutdatedDeviceLists();
+            expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
+
+            queryDefer3.resolve(utils.deepCopy(signedDeviceList));
+
+            // allow promise chain to complete
+            return prom3;
+        }).then(() => {
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
+        });
+    });
+});

spec/unit/crypto/algorithms/megolm.spec.js

@@ -0,0 +1,267 @@
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run megolm tests: libolm not available");
+}
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import sdk from '../../../..';
+import algorithms from '../../../../lib/crypto/algorithms';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+import Crypto from '../../../../lib/crypto';
+
+const MatrixEvent = sdk.MatrixEvent;
+const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
+
+const ROOM_ID = '!ROOM:ID';
+
+const Olm = global.Olm;
+
+describe("MegolmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let megolmDecryption;
+    let mockOlmLib;
+    let mockCrypto;
+    let mockBaseApis;
+
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        mockCrypto = testUtils.mock(Crypto, 'Crypto');
+        mockBaseApis = {};
+
+        const mockStorage = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(mockStorage);
+        const cryptoStore = new MemoryCryptoStore(mockStorage);
+
+        const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+
+        megolmDecryption = new MegolmDecryption({
+            userId: '@user:id',
+            crypto: mockCrypto,
+            olmDevice: olmDevice,
+            baseApis: mockBaseApis,
+            roomId: ROOM_ID,
+        });
+
+
+        // we stub out the olm encryption bits
+        mockOlmLib = {};
+        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.encryptMessageForDevice =
+            expect.createSpy().andReturn(Promise.resolve());
+        megolmDecryption.olmlib = mockOlmLib;
+    });
+
+    describe('receives some keys:', function() {
+        let groupSession;
+        beforeEach(async function() {
+            await Olm.init();
+            groupSession = new global.Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // construct a fake decrypted key event via the use of a mocked
+            // 'crypto' implementation.
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+            });
+            const decryptedData = {
+                clearEvent: {
+                    type: 'm.room_key',
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        room_id: ROOM_ID,
+                        session_id: groupSession.session_id(),
+                        session_key: groupSession.session_key(),
+                    },
+                },
+                senderCurve25519Key: "SENDER_CURVE25519",
+                claimedEd25519Key: "SENDER_ED25519",
+            };
+
+            const mockCrypto = {
+                decryptEvent: function() {
+                    return Promise.resolve(decryptedData);
+                },
+            };
+
+            await event.attemptDecryption(mockCrypto).then(() => {
+                megolmDecryption.onRoomKeyEvent(event);
+            });
+        });
+
+        it('can decrypt an event', function() {
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                    ciphertext: groupSession.encrypt(JSON.stringify({
+                        room_id: ROOM_ID,
+                        content: 'testytest',
+                    })),
+                },
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
+            });
+        });
+
+        it('can respond to a key request event', function() {
+            const keyRequest = {
+                userId: '@alice:foo',
+                deviceId: 'alidevice',
+                requestBody: {
+                    room_id: ROOM_ID,
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                },
+            };
+
+            return megolmDecryption.hasKeysForKeyRequest(
+                keyRequest,
+            ).then((hasKeys) => {
+                expect(hasKeys).toBe(true);
+
+                // set up some pre-conditions for the share call
+                const deviceInfo = {};
+                mockCrypto.getStoredDevice.andReturn(deviceInfo);
+
+                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
+                    Promise.resolve({'@alice:foo': {'alidevice': {
+                        sessionId: 'alisession',
+                    }}}),
+                );
+
+                const awaitEncryptForDevice = new Promise((res, rej) => {
+                    mockOlmLib.encryptMessageForDevice.andCall(() => {
+                        res();
+                        return Promise.resolve();
+                    });
+                });
+
+                mockBaseApis.sendToDevice = expect.createSpy();
+
+                // do the share
+                megolmDecryption.shareKeysWithDevice(keyRequest);
+
+                // it's asynchronous, so we have to wait a bit
+                return awaitEncryptForDevice;
+            }).then(() => {
+                // check that it called encryptMessageForDevice with
+                // appropriate args.
+                expect(mockOlmLib.encryptMessageForDevice.calls.length)
+                    .toEqual(1);
+
+                const call = mockOlmLib.encryptMessageForDevice.calls[0];
+                const payload = call.arguments[6];
+
+                expect(payload.type).toEqual("m.forwarded_room_key");
+                expect(payload.content).toInclude({
+                    sender_key: "SENDER_CURVE25519",
+                    sender_claimed_ed25519_key: "SENDER_ED25519",
+                    session_id: groupSession.session_id(),
+                    chain_index: 0,
+                    forwarding_curve25519_key_chain: [],
+                });
+                expect(payload.content.session_key).toExist();
+            });
+        });
+
+        it("can detect replay attacks", function() {
+            // trying to decrypt two different messages (marked by different
+            // event IDs or timestamps) using the same (sender key, session id,
+            // message index) triple should result in an exception being thrown
+            // as it should be detected as a replay attack.
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event1 = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            const successHandler = expect.createSpy();
+            const failureHandler = expect.createSpy()
+                .andCall((err) => {
+                    expect(err.toString()).toMatch(
+                        /Duplicate message index, possible replay attack/,
+                    );
+                });
+
+            return megolmDecryption.decryptEvent(event1).then((res) => {
+                const event2 = new MatrixEvent({
+                    type: 'm.room.encrypted',
+                    room_id: ROOM_ID,
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        sender_key: "SENDER_CURVE25519",
+                        session_id: sessionId,
+                        ciphertext: cipherText,
+                    },
+                    event_id: "$event2",
+                    origin_server_ts: 1507754149000,
+                });
+
+                return megolmDecryption.decryptEvent(event2);
+            }).then(
+                successHandler,
+                failureHandler,
+            ).then(() => {
+                expect(successHandler).toNotHaveBeenCalled();
+                expect(failureHandler).toHaveBeenCalled();
+            });
+        });
+
+        it("allows re-decryption of the same event", function() {
+            // in contrast with the previous test, if the event ID and
+            // timestamp are the same, then it should not be considered a
+            // replay attack
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                return megolmDecryption.decryptEvent(event);
+                // test is successful if no exception is thrown
+            });
+        });
+    });
+});

spec/unit/crypto/algorithms/olm.spec.js

@@ -0,0 +1,90 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run megolm tests: libolm not available");
+}
+
+import expect from 'expect';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+
+function makeOlmDevice() {
+    const mockStorage = new MockStorageApi();
+    const sessionStore = new WebStorageSessionStore(mockStorage);
+    const cryptoStore = new MemoryCryptoStore(mockStorage);
+    const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    return olmDevice;
+}
+
+async function setupSession(initiator, opponent) {
+    await opponent.generateOneTimeKeys(1);
+    const keys = await opponent.getOneTimeKeys();
+    const firstKey = Object.values(keys['curve25519'])[0];
+
+    const sid = await initiator.createOutboundSession(
+        opponent.deviceCurve25519Key, firstKey,
+    );
+    return sid;
+}
+
+describe("OlmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let aliceOlmDevice;
+    let bobOlmDevice;
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        await global.Olm.init();
+
+        aliceOlmDevice = makeOlmDevice();
+        bobOlmDevice = makeOlmDevice();
+        await aliceOlmDevice.init();
+        await bobOlmDevice.init();
+    });
+
+    describe('olm', function() {
+        it("can decrypt messages", async function() {
+            const sid = await setupSession(aliceOlmDevice, bobOlmDevice);
+
+            const ciphertext = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sid,
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+
+            const result = await bobOlmDevice.createInboundSession(
+                aliceOlmDevice.deviceCurve25519Key,
+                ciphertext.type,
+                ciphertext.body,
+            );
+            expect(result.payload).toEqual(
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+        });
+    });
+});

spec/unit/event-timeline.spec.js

@@ -1,32 +1,37 @@
 "use strict";
-var sdk = require("../..");
-var EventTimeline = sdk.EventTimeline;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const EventTimeline = sdk.EventTimeline;
+const utils = require("../test-utils");
 
 function mockRoomStates(timeline) {
     timeline._startState = utils.mock(sdk.RoomState, "startState");
     timeline._endState = utils.mock(sdk.RoomState, "endState");
 }
 
+import expect from 'expect';
+
 describe("EventTimeline", function() {
-    var roomId = "!foo:bar";
-    var userA = "@alice:bar";
-    var userB = "@bertha:bar";
-    var timeline;
+    const roomId = "!foo:bar";
+    const userA = "@alice:bar";
+    const userB = "@bertha:bar";
+    let timeline;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
 
         // XXX: this is a horrid hack; should use sinon or something instead to mock
-        var timelineSet = { room: { roomId: roomId }};
-        timelineSet.room.getUnfilteredTimelineSet = function() { return timelineSet; };
+        const timelineSet = { room: { roomId: roomId }};
+        timelineSet.room.getUnfilteredTimelineSet = function() {
+            return timelineSet;
+        };
 
         timeline = new EventTimeline(timelineSet);
     });
 
     describe("construction", function() {
         it("getRoomId should get room id", function() {
-            var v = timeline.getRoomId();
+            const v = timeline.getRoomId();
             expect(v).toEqual(roomId);
         });
     });
@@ -37,7 +42,7 @@ describe("EventTimeline", function() {
         });
 
         it("should copy state events to start and end state", function() {
-            var events = [
+            const events = [
                 utils.mkMembership({
                     room: roomId, mship: "invite", user: userB, skey: userA,
                     event: true,
@@ -46,34 +51,38 @@ describe("EventTimeline", function() {
                     type: "m.room.name", room: roomId, user: userB,
                     event: true,
                     content: { name: "New room" },
-                })
+                }),
             ];
             timeline.initialiseState(events);
             expect(timeline._startState.setStateEvents).toHaveBeenCalledWith(
-                events
+                events,
             );
             expect(timeline._endState.setStateEvents).toHaveBeenCalledWith(
-                events
+                events,
             );
         });
 
         it("should raise an exception if called after events are added", function() {
-            var event =
+            const event =
                 utils.mkMessage({
                     room: roomId, user: userA, msg: "Adam stole the plushies",
                     event: true,
                 });
 
-            var state = [
+            const state = [
                 utils.mkMembership({
                     room: roomId, mship: "invite", user: userB, skey: userA,
                     event: true,
-                })
+                }),
             ];
 
-            expect(function() { timeline.initialiseState(state); }).not.toThrow();
+            expect(function() {
+                timeline.initialiseState(state);
+            }).toNotThrow();
             timeline.addEvent(event, false);
-            expect(function() { timeline.initialiseState(state); }).toThrow();
+            expect(function() {
+                timeline.initialiseState(state);
+            }).toThrow();
         });
     });
 
@@ -99,8 +108,8 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should set neighbour", function() {
-            var prev = {a: "a"};
-            var next = {b: "b"};
+            const prev = {a: "a"};
+            const next = {b: "b"};
             timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
             timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS)).toBe(prev);
@@ -108,11 +117,11 @@ describe("EventTimeline", function() {
         });
 
         it("setNeighbouringTimeline should throw if called twice", function() {
-            var prev = {a: "a"};
-            var next = {b: "b"};
+            const prev = {a: "a"};
+            const next = {b: "b"};
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS))
                 .toBe(prev);
             expect(function() {
@@ -121,7 +130,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
-            }).not.toThrow();
+            }).toNotThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.FORWARDS))
                 .toBe(next);
             expect(function() {
@@ -135,7 +144,7 @@ describe("EventTimeline", function() {
             mockRoomStates(timeline);
         });
 
-        var events = [
+        const events = [
             utils.mkMessage({
                 room: roomId, user: userA, msg: "hungry hungry hungry",
                 event: true,
@@ -148,7 +157,7 @@ describe("EventTimeline", function() {
 
         it("should be able to add events to the end", function() {
             timeline.addEvent(events[0], false);
-            var initialIndex = timeline.getBaseIndex();
+            const initialIndex = timeline.getBaseIndex();
             timeline.addEvent(events[1], false);
             expect(timeline.getBaseIndex()).toEqual(initialIndex);
             expect(timeline.getEvents().length).toEqual(2);
@@ -158,7 +167,7 @@ describe("EventTimeline", function() {
 
         it("should be able to add events to the start", function() {
             timeline.addEvent(events[0], true);
-            var initialIndex = timeline.getBaseIndex();
+            const initialIndex = timeline.getBaseIndex();
             timeline.addEvent(events[1], true);
             expect(timeline.getBaseIndex()).toEqual(initialIndex + 1);
             expect(timeline.getEvents().length).toEqual(2);
@@ -167,38 +176,38 @@ describe("EventTimeline", function() {
         });
 
         it("should set event.sender for new and old events", function() {
-            var sentinel = {
+            const sentinel = {
                 userId: userA,
                 membership: "join",
-                name: "Alice"
+                name: "Alice",
             };
-            var oldSentinel = {
+            const oldSentinel = {
                 userId: userA,
                 membership: "join",
-                name: "Old Alice"
+                name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
                     return null;
                 });
 
-            var newEv = utils.mkEvent({
+            const newEv = utils.mkEvent({
                 type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "New Room Name" }
+                content: { name: "New Room Name" },
             });
-            var oldEv = utils.mkEvent({
+            const oldEv = utils.mkEvent({
                 type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "Old Room Name" }
+                content: { name: "Old Room Name" },
             });
 
             timeline.addEvent(newEv, false);
@@ -209,36 +218,36 @@ describe("EventTimeline", function() {
 
         it("should set event.target for new and old m.room.member events",
         function() {
-            var sentinel = {
+            const sentinel = {
                 userId: userA,
                 membership: "join",
-                name: "Alice"
+                name: "Alice",
             };
-            var oldSentinel = {
+            const oldSentinel = {
                 userId: userA,
                 membership: "join",
-                name: "Old Alice"
+                name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCallFake(function(uid) {
+                .andCall(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
                     return null;
                 });
 
-            var newEv = utils.mkMembership({
-                room: roomId, mship: "invite", user: userB, skey: userA, event: true
+            const newEv = utils.mkMembership({
+                room: roomId, mship: "invite", user: userB, skey: userA, event: true,
             });
-            var oldEv = utils.mkMembership({
-                room: roomId, mship: "ban", user: userB, skey: userA, event: true
+            const oldEv = utils.mkMembership({
+                room: roomId, mship: "ban", user: userB, skey: userA, event: true,
             });
             timeline.addEvent(newEv, false);
             expect(newEv.target).toEqual(sentinel);
@@ -248,16 +257,16 @@ describe("EventTimeline", function() {
 
         it("should call setStateEvents on the right RoomState with the right " +
            "forwardLooking value for new events", function() {
-            var events = [
+            const events = [
                 utils.mkMembership({
-                    room: roomId, mship: "invite", user: userB, skey: userA, event: true
+                    room: roomId, mship: "invite", user: userB, skey: userA, event: true,
                 }),
                 utils.mkEvent({
                     type: "m.room.name", room: roomId, user: userB, event: true,
                     content: {
-                        name: "New room"
-                    }
-                })
+                        name: "New room",
+                    },
+                }),
             ];
 
             timeline.addEvent(events[0], false);
@@ -272,22 +281,22 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(true);
 
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
         });
 
 
         it("should call setStateEvents on the right RoomState with the right " +
            "forwardLooking value for old events", function() {
-            var events = [
+            const events = [
                 utils.mkMembership({
-                    room: roomId, mship: "invite", user: userB, skey: userA, event: true
+                    room: roomId, mship: "invite", user: userB, skey: userA, event: true,
                 }),
                 utils.mkEvent({
                     type: "m.room.name", room: roomId, user: userB, event: true,
                     content: {
-                        name: "New room"
-                    }
-                })
+                        name: "New room",
+                    },
+                }),
             ];
 
             timeline.addEvent(events[0], true);
@@ -302,12 +311,12 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(false);
 
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                not.toHaveBeenCalled();
+                toNotHaveBeenCalled();
         });
     });
 
     describe("removeEvent", function() {
-        var events = [
+        const events = [
             utils.mkMessage({
                 room: roomId, user: userA, msg: "hungry hungry hungry",
                 event: true,
@@ -327,7 +336,7 @@ describe("EventTimeline", function() {
             timeline.addEvent(events[1], false);
             expect(timeline.getEvents().length).toEqual(2);
 
-            var ev = timeline.removeEvent(events[0].getId());
+            let ev = timeline.removeEvent(events[0].getId());
             expect(ev).toBe(events[0]);
             expect(timeline.getEvents().length).toEqual(1);
 
@@ -359,7 +368,7 @@ describe("EventTimeline", function() {
            function() {
                timeline.addEvent(events[0], true);
                timeline.removeEvent(events[0].getId());
-               var initialIndex = timeline.getBaseIndex();
+               const initialIndex = timeline.getBaseIndex();
                timeline.addEvent(events[1], false);
                timeline.addEvent(events[2], false);
                expect(timeline.getBaseIndex()).toEqual(initialIndex);
@@ -367,4 +376,3 @@ describe("EventTimeline", function() {
            });
     });
 });
-

spec/unit/event.spec.js

@@ -0,0 +1,82 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import sdk from '../..';
+const MatrixEvent = sdk.MatrixEvent;
+
+import testUtils from '../test-utils';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+describe("MatrixEvent", () => {
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+    });
+
+    describe(".attemptDecryption", () => {
+        let encryptedEvent;
+
+        beforeEach(() => {
+            encryptedEvent = new MatrixEvent({
+                id: 'test_encrypted_event',
+                type: 'm.room.encrypted',
+                content: {
+                    ciphertext: 'secrets',
+                },
+            });
+        });
+
+        it('should retry decryption if a retry is queued', () => {
+            let callCount = 0;
+
+            let prom2;
+
+            const crypto = {
+                decryptEvent: function() {
+                    ++callCount;
+                    console.log(`decrypt: ${callCount}`);
+                    if (callCount == 1) {
+                        // schedule a second decryption attempt while
+                        // the first one is still running.
+                        prom2 = encryptedEvent.attemptDecryption(crypto);
+
+                        const error = new Error("nope");
+                        error.name = 'DecryptionError';
+                        return Promise.reject(error);
+                    } else {
+                        expect(prom2.isFulfilled()).toBe(
+                            false, 'second attemptDecryption resolved too soon');
+
+                        return Promise.resolve({
+                            clearEvent: {
+                                type: 'm.room.message',
+                            },
+                        });
+                    }
+                },
+            };
+
+            return encryptedEvent.attemptDecryption(crypto).then(() => {
+                expect(callCount).toEqual(2);
+                expect(encryptedEvent.getType()).toEqual('m.room.message');
+
+                // make sure the second attemptDecryption resolves
+                return prom2;
+            });
+        });
+    });
+});

spec/unit/filter.spec.js

@@ -1,12 +1,15 @@
 "use strict";
-var sdk = require("../..");
-var Filter = sdk.Filter;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const Filter = sdk.Filter;
+const utils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("Filter", function() {
-    var filterId = "f1lt3ring15g00d4ursoul";
-    var userId = "@sir_arthur_david:humming.tiger";
-    var filter;
+    const filterId = "f1lt3ring15g00d4ursoul";
+    const userId = "@sir_arthur_david:humming.tiger";
+    let filter;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -15,10 +18,10 @@ describe("Filter", function() {
 
     describe("fromJson", function() {
         it("create a new Filter from the provided values", function() {
-            var definition = {
-                event_fields: ["type", "content"]
+            const definition = {
+                event_fields: ["type", "content"],
             };
-            var f = Filter.fromJson(userId, filterId, definition);
+            const f = Filter.fromJson(userId, filterId, definition);
             expect(f.getDefinition()).toEqual(definition);
             expect(f.userId).toEqual(userId);
             expect(f.filterId).toEqual(filterId);
@@ -31,17 +34,17 @@ describe("Filter", function() {
             expect(filter.getDefinition()).toEqual({
                 room: {
                     timeline: {
-                        limit: 10
-                    }
-                }
+                        limit: 10,
+                    },
+                },
             });
         });
     });
 
     describe("setDefinition/getDefinition", function() {
         it("should set and get the filter body", function() {
-            var definition = {
-                event_format: "client"
+            const definition = {
+                event_format: "client",
             };
             filter.setDefinition(definition);
             expect(filter.getDefinition()).toEqual(definition);

spec/unit/interactive-auth.spec.js

@@ -15,12 +15,23 @@ limitations under the License.
 */
 "use strict";
 
-var q = require("q");
-var sdk = require("../..");
-var utils = require("../test-utils");
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const utils = require("../test-utils");
 
-var InteractiveAuth = sdk.InteractiveAuth;
-var MatrixError = sdk.MatrixError;
+const InteractiveAuth = sdk.InteractiveAuth;
+const MatrixError = sdk.MatrixError;
+
+import expect from 'expect';
+
+// Trivial client object to test interactive auth
+// (we do not need TestClient here)
+class FakeClient {
+    generateClientSecret() {
+        return "testcl1Ent5EcreT";
+    }
+}
 
 describe("InteractiveAuth", function() {
     beforeEach(function() {
@@ -28,12 +39,13 @@ describe("InteractiveAuth", function() {
     });
 
     it("should start an auth stage and complete it", function(done) {
-        var doRequest = jasmine.createSpy('doRequest');
-        var startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();
 
-        var ia = new InteractiveAuth({
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
             doRequest: doRequest,
-            startAuthStage: startAuthStage,
+            stateUpdated: stateUpdated,
             authData: {
                 session: "sessionId",
                 flows: [
@@ -51,7 +63,8 @@ describe("InteractiveAuth", function() {
         });
 
         // first we expect a call here
-        startAuthStage.andCallFake(function(stage) {
+        stateUpdated.andCall(function(stage) {
+            console.log('aaaa');
             expect(stage).toEqual("logintype");
             ia.submitAuthDict({
                 type: "logintype",
@@ -60,40 +73,42 @@ describe("InteractiveAuth", function() {
         });
 
         // .. which should trigger a call here
-        var requestRes = {"a": "b"};
-        doRequest.andCallFake(function(authData) {
+        const requestRes = {"a": "b"};
+        doRequest.andCall(function(authData) {
+            console.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
                 type: "logintype",
                 foo: "bar",
             });
-            return q(requestRes);
+            return Promise.resolve(requestRes);
         });
 
         ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(1);
-            expect(startAuthStage.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+            expect(stateUpdated.calls.length).toEqual(1);
+        }).nodeify(done);
     });
 
     it("should make a request if no authdata is provided", function(done) {
-        var doRequest = jasmine.createSpy('doRequest');
-        var startAuthStage = jasmine.createSpy('startAuthStage');
+        const doRequest = expect.createSpy();
+        const stateUpdated = expect.createSpy();
 
-        var ia = new InteractiveAuth({
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            stateUpdated: stateUpdated,
             doRequest: doRequest,
-            startAuthStage: startAuthStage,
         });
 
         expect(ia.getSessionId()).toBe(undefined);
         expect(ia.getStageParams("logintype")).toBe(undefined);
 
         // first we expect a call to doRequest
-        doRequest.andCallFake(function(authData) {
+        doRequest.andCall(function(authData) {
             console.log("request1", authData);
-            expect(authData).toBe(null);
-            var err = new MatrixError({
+            expect(authData).toEqual({});
+            const err = new MatrixError({
                 session: "sessionId",
                 flows: [
                     { stages: ["logintype"] },
@@ -106,9 +121,9 @@ describe("InteractiveAuth", function() {
             throw err;
         });
 
-        // .. which should be followed by a call to startAuthStage
-        var requestRes = {"a": "b"};
-        startAuthStage.andCallFake(function(stage) {
+        // .. which should be followed by a call to stateUpdated
+        const requestRes = {"a": "b"};
+        stateUpdated.andCall(function(stage) {
             expect(stage).toEqual("logintype");
             expect(ia.getSessionId()).toEqual("sessionId");
             expect(ia.getStageParams("logintype")).toEqual({
@@ -116,14 +131,14 @@ describe("InteractiveAuth", function() {
             });
 
             // submitAuthDict should trigger another call to doRequest
-            doRequest.andCallFake(function(authData) {
+            doRequest.andCall(function(authData) {
                 console.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
                     type: "logintype",
                     foo: "bar",
                 });
-                return q(requestRes);
+                return Promise.resolve(requestRes);
             });
 
             ia.submitAuthDict({
@@ -135,7 +150,7 @@ describe("InteractiveAuth", function() {
         ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
             expect(doRequest.calls.length).toEqual(2);
-            expect(startAuthStage.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+            expect(stateUpdated.calls.length).toEqual(1);
+        }).nodeify(done);
     });
 });

spec/unit/matrix-client.spec.js

@@ -1,44 +1,51 @@
 "use strict";
-var q = require("q");
-var sdk = require("../..");
-var MatrixClient = sdk.MatrixClient;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const MatrixClient = sdk.MatrixClient;
+const utils = require("../test-utils");
+
+import expect from 'expect';
+import lolex from 'lolex';
 
 describe("MatrixClient", function() {
-    var userId = "@alice:bar";
-    var identityServerUrl = "https://identity.server";
-    var identityServerDomain = "identity.server";
-    var client, store, scheduler;
+    const userId = "@alice:bar";
+    const identityServerUrl = "https://identity.server";
+    const identityServerDomain = "identity.server";
+    let client;
+    let store;
+    let scheduler;
+    let clock;
 
-    var KEEP_ALIVE_PATH = "/_matrix/client/versions";
+    const KEEP_ALIVE_PATH = "/_matrix/client/versions";
 
-    var PUSH_RULES_RESPONSE = {
+    const PUSH_RULES_RESPONSE = {
         method: "GET",
         path: "/pushrules/",
-        data: {}
+        data: {},
     };
 
-    var FILTER_PATH = "/user/" + encodeURIComponent(userId) + "/filter";
+    const FILTER_PATH = "/user/" + encodeURIComponent(userId) + "/filter";
 
-    var FILTER_RESPONSE = {
+    const FILTER_RESPONSE = {
         method: "POST",
         path: FILTER_PATH,
-        data: { filter_id: "f1lt3r" }
+        data: { filter_id: "f1lt3r" },
     };
 
-    var SYNC_DATA = {
+    const SYNC_DATA = {
         next_batch: "s_5_3",
         presence: { events: [] },
-        rooms: {}
+        rooms: {},
     };
 
-    var SYNC_RESPONSE = {
+    const SYNC_RESPONSE = {
         method: "GET",
         path: "/sync",
-        data: SYNC_DATA
+        data: SYNC_DATA,
     };
 
-    var httpLookups = [
+    let httpLookups = [
         // items are objects which look like:
         // {
         //   method: "GET",
@@ -51,14 +58,14 @@ describe("MatrixClient", function() {
         // }
         // items are popped off when processed and block if no items left.
     ];
-    var accept_keepalives;
-    var pendingLookup = null;
+    let acceptKeepalives;
+    let pendingLookup = null;
     function httpReq(cb, method, path, qp, data, prefix) {
-        if (path === KEEP_ALIVE_PATH && accept_keepalives) {
-            return q();
+        if (path === KEEP_ALIVE_PATH && acceptKeepalives) {
+            return Promise.resolve();
         }
-        var next = httpLookups.shift();
-        var logLine = (
+        const next = httpLookups.shift();
+        const logLine = (
             "MatrixClient[UT] RECV " + method + " " + path + "  " +
             "EXPECT " + (next ? next.method : next) + " " + (next ? next.path : next)
         );
@@ -73,20 +80,20 @@ describe("MatrixClient", function() {
                 expect(false).toBe(
                     true, ">1 pending request. You should probably handle them. " +
                     "PENDING: " + JSON.stringify(pendingLookup) + " JUST GOT: " +
-                    method + " " + path
+                    method + " " + path,
                 );
             }
             pendingLookup = {
-                promise: q.defer().promise,
+                promise: Promise.defer().promise,
                 method: method,
-                path: path
+                path: path,
             };
             return pendingLookup.promise;
         }
         if (next.path === path && next.method === method) {
             console.log(
                 "MatrixClient[UT] Matched. Returning " +
-                (next.error ? "BAD" : "GOOD") + " response"
+                (next.error ? "BAD" : "GOOD") + " response",
             );
             if (next.expectBody) {
                 expect(next.expectBody).toEqual(data);
@@ -102,32 +109,39 @@ describe("MatrixClient", function() {
             }
 
             if (next.error) {
-                return q.reject({
+                return Promise.reject({
                     errcode: next.error.errcode,
                     httpStatus: next.error.httpStatus,
                     name: next.error.errcode,
                     message: "Expected testing error",
-                    data: next.error
+                    data: next.error,
                 });
             }
-            return q(next.data);
+            return Promise.resolve(next.data);
         }
         expect(true).toBe(false, "Expected different request. " + logLine);
-        return q.defer().promise;
+        return Promise.defer().promise;
     }
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
-        scheduler = jasmine.createSpyObj("scheduler", [
+        clock = lolex.install();
+        scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
-            "setProcessFunction"
-        ]);
-        store = jasmine.createSpyObj("store", [
+            "setProcessFunction",
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store = [
             "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "setSyncToken", "storeEvents", "storeRoom", "storeUser",
-            "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter"
-        ]);
+            "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
+            "getSyncAccumulator", "startup", "deleteAllData",
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
+        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.storeClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.isNewlyCreated = expect.createSpy().andReturn(Promise.resolve(true));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",
             idBaseUrl: identityServerUrl,
@@ -135,20 +149,20 @@ describe("MatrixClient", function() {
             request: function() {}, // NOP
             store: store,
             scheduler: scheduler,
-            userId: userId
+            userId: userId,
         });
         // FIXME: We shouldn't be yanking _http like this.
-        client._http = jasmine.createSpyObj("httpApi", [
+        client._http = [
             "authedRequest", "authedRequestWithPrefix", "getContentUri",
-            "request", "requestWithPrefix", "uploadContent"
-        ]);
-        client._http.authedRequest.andCallFake(httpReq);
-        client._http.authedRequestWithPrefix.andCallFake(httpReq);
-        client._http.requestWithPrefix.andCallFake(httpReq);
-        client._http.request.andCallFake(httpReq);
+            "request", "requestWithPrefix", "uploadContent",
+        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        client._http.authedRequest.andCall(httpReq);
+        client._http.authedRequestWithPrefix.andCall(httpReq);
+        client._http.requestWithPrefix.andCall(httpReq);
+        client._http.request.andCall(httpReq);
 
         // set reasonable working defaults
-        accept_keepalives = true;
+        acceptKeepalives = true;
         pendingLookup = null;
         httpLookups = [];
         httpLookups.push(PUSH_RULES_RESPONSE);
@@ -157,54 +171,61 @@ describe("MatrixClient", function() {
     });
 
     afterEach(function() {
+        clock.uninstall();
         // need to re-stub the requests with NOPs because there are no guarantees
         // clients from previous tests will be GC'd before the next test. This
         // means they may call /events and then fail an expect() which will fail
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
-        client._http.authedRequest.andCallFake(function() {
-            return q.defer().promise;
+        client._http.authedRequest.andCall(function() {
+            return Promise.defer().promise;
         });
-        client._http.authedRequestWithPrefix.andCallFake(function() {
-            return q.defer().promise;
+        client._http.authedRequestWithPrefix.andCall(function() {
+            return Promise.defer().promise;
         });
     });
 
-    it("should not POST /filter if a matching filter already exists", function(done) {
+    it("should not POST /filter if a matching filter already exists", async function() {
         httpLookups = [];
         httpLookups.push(PUSH_RULES_RESPONSE);
         httpLookups.push(SYNC_RESPONSE);
-        var filterId = "ehfewf";
+        const filterId = "ehfewf";
         store.getFilterIdByName.andReturn(filterId);
-        var filter = new sdk.Filter(0, filterId);
+        const filter = new sdk.Filter(0, filterId);
         filter.setDefinition({"room": {"timeline": {"limit": 8}}});
         store.getFilter.andReturn(filter);
-        client.startClient();
-
-        client.on("sync", function syncListener(state) {
-            if (state === "SYNCING") {
-                expect(httpLookups.length).toEqual(0);
-                client.removeListener("sync", syncListener);
-                done();
-            }
+        const syncPromise = new Promise((resolve, reject) => {
+            client.on("sync", function syncListener(state) {
+                if (state === "SYNCING") {
+                    expect(httpLookups.length).toEqual(0);
+                    client.removeListener("sync", syncListener);
+                    resolve();
+                } else if (state === "ERROR") {
+                    reject(new Error("sync error"));
+                }
+            });
         });
+        await client.startClient();
+        await syncPromise;
     });
 
     describe("getSyncState", function() {
-
         it("should return null if the client isn't started", function() {
-            expect(client.getSyncState()).toBeNull();
+            expect(client.getSyncState()).toBe(null);
         });
 
-        it("should return the same sync state as emitted sync events", function(done) {
-            client.on("sync", function syncListener(state) {
-                expect(state).toEqual(client.getSyncState());
-                if (state === "SYNCING") {
-                    client.removeListener("sync", syncListener);
-                    done();
-                }
+        it("should return the same sync state as emitted sync events", async function() {
+            const syncingPromise = new Promise((resolve) => {
+                client.on("sync", function syncListener(state) {
+                    expect(state).toEqual(client.getSyncState());
+                    if (state === "SYNCING") {
+                        client.removeListener("sync", syncListener);
+                        resolve();
+                    }
+                });
             });
-            client.startClient();
+            await client.startClient();
+            await syncingPromise;
         });
     });
 
@@ -219,7 +240,7 @@ describe("MatrixClient", function() {
                 // and they all need to be stored!
                 return "FILTER_SYNC_" + userId + (suffix ? "_" + suffix : "");
             }
-            var invalidFilterId = 'invalidF1lt3r';
+            const invalidFilterId = 'invalidF1lt3r';
             httpLookups = [];
             httpLookups.push({
                 method: "GET",
@@ -229,15 +250,15 @@ describe("MatrixClient", function() {
                     name: "M_UNKNOWN",
                     message: "No row found",
                     data: { errcode: "M_UNKNOWN", error: "No row found" },
-                    httpStatus: 404
-                }
+                    httpStatus: 404,
+                },
             });
             httpLookups.push(FILTER_RESPONSE);
             store.getFilterIdByName.andReturn(invalidFilterId);
 
-            var filterName = getFilterName(client.credentials.userId);
+            const filterName = getFilterName(client.credentials.userId);
             client.store.setFilterIdByName(filterName, invalidFilterId);
-            var filter = new sdk.Filter(client.credentials.userId);
+            const filter = new sdk.Filter(client.credentials.userId);
 
             client.getOrCreateFilter(filterName, filter).then(function(filterId) {
                 expect(filterId).toEqual(FILTER_RESPONSE.data.filter_id);
@@ -247,8 +268,8 @@ describe("MatrixClient", function() {
     });
 
     describe("retryImmediately", function() {
-        it("should return false if there is no request waiting", function() {
-            client.startClient();
+        it("should return false if there is no request waiting", async function() {
+            await client.startClient();
             expect(client.retryImmediately()).toBe(false);
         });
 
@@ -256,7 +277,7 @@ describe("MatrixClient", function() {
             httpLookups = [];
             httpLookups.push(PUSH_RULES_RESPONSE);
             httpLookups.push({
-                method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" }
+                method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" },
             });
             httpLookups.push(FILTER_RESPONSE);
             httpLookups.push(SYNC_RESPONSE);
@@ -265,7 +286,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(2);
                     expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -279,21 +300,21 @@ describe("MatrixClient", function() {
 
         it("should work on /sync", function(done) {
             httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" }
+                method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" },
             });
             httpLookups.push({
-                method: "GET", path: "/sync", data: SYNC_DATA
+                method: "GET", path: "/sync", data: SYNC_DATA,
             });
 
             client.on("sync", function syncListener(state) {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(1);
                     expect(client.retryImmediately()).toBe(
-                        true, "retryImmediately returned false"
+                        true, "retryImmediately returned false",
                     );
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "RECONNECTING" && httpLookups.length > 0) {
-                    jasmine.Clock.tick(10000);
+                    clock.tick(10000);
                 } else if (state === "SYNCING" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -305,7 +326,7 @@ describe("MatrixClient", function() {
         it("should work on /pushrules", function(done) {
             httpLookups = [];
             httpLookups.push({
-                method: "GET", path: "/pushrules/", error: { errcode: "NOPE_NOPE_NOPE" }
+                method: "GET", path: "/pushrules/", error: { errcode: "NOPE_NOPE_NOPE" },
             });
             httpLookups.push(PUSH_RULES_RESPONSE);
             httpLookups.push(FILTER_RESPONSE);
@@ -315,7 +336,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(3);
                     expect(client.retryImmediately()).toBe(true);
-                    jasmine.Clock.tick(1);
+                    clock.tick(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -329,12 +350,11 @@ describe("MatrixClient", function() {
     });
 
     describe("emitted sync events", function() {
-
         function syncChecker(expectedStates, done) {
             return function syncListener(state, old) {
-                var expected = expectedStates.shift();
+                const expected = expectedStates.shift();
                 console.log(
-                    "'sync' curr=%s old=%s EXPECT=%s", state, old, expected
+                    "'sync' curr=%s old=%s EXPECT=%s", state, old, expected,
                 );
                 if (!expected) {
                     done();
@@ -347,58 +367,59 @@ describe("MatrixClient", function() {
                     done();
                 }
                 // standard retry time is 5 to 10 seconds
-                jasmine.Clock.tick(10000);
+                clock.tick(10000);
             };
         }
 
         it("should transition null -> PREPARED after the first /sync", function(done) {
-            var expectedStates = [];
+            const expectedStates = [];
             expectedStates.push(["PREPARED", null]);
             client.on("sync", syncChecker(expectedStates, done));
             client.startClient();
         });
 
         it("should transition null -> ERROR after a failed /filter", function(done) {
-            var expectedStates = [];
+            const expectedStates = [];
             httpLookups = [];
             httpLookups.push(PUSH_RULES_RESPONSE);
             httpLookups.push({
-                method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" }
+                method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" },
             });
             expectedStates.push(["ERROR", null]);
             client.on("sync", syncChecker(expectedStates, done));
             client.startClient();
         });
 
-        it("should transition ERROR -> PREPARED after /sync if prev failed",
+        it("should transition ERROR -> CATCHUP after /sync if prev failed",
         function(done) {
-            var expectedStates = [];
-            accept_keepalives = false;
+            const expectedStates = [];
+            acceptKeepalives = false;
             httpLookups = [];
             httpLookups.push(PUSH_RULES_RESPONSE);
             httpLookups.push(FILTER_RESPONSE);
             httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" }
+                method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" },
             });
             httpLookups.push({
-                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+                method: "GET", path: KEEP_ALIVE_PATH,
+                error: { errcode: "KEEPALIVE_FAIL" },
             });
             httpLookups.push({
-                method: "GET", path: KEEP_ALIVE_PATH, data: {}
+                method: "GET", path: KEEP_ALIVE_PATH, data: {},
             });
             httpLookups.push({
-                method: "GET", path: "/sync", data: SYNC_DATA
+                method: "GET", path: "/sync", data: SYNC_DATA,
             });
 
             expectedStates.push(["RECONNECTING", null]);
             expectedStates.push(["ERROR", "RECONNECTING"]);
-            expectedStates.push(["PREPARED", "ERROR"]);
+            expectedStates.push(["CATCHUP", "ERROR"]);
             client.on("sync", syncChecker(expectedStates, done));
             client.startClient();
         });
 
         it("should transition PREPARED -> SYNCING after /sync", function(done) {
-            var expectedStates = [];
+            const expectedStates = [];
             expectedStates.push(["PREPARED", null]);
             expectedStates.push(["SYNCING", "PREPARED"]);
             client.on("sync", syncChecker(expectedStates, done));
@@ -406,13 +427,14 @@ describe("MatrixClient", function() {
         });
 
         it("should transition SYNCING -> ERROR after a failed /sync", function(done) {
-            accept_keepalives = false;
-            var expectedStates = [];
+            acceptKeepalives = false;
+            const expectedStates = [];
             httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
+                method: "GET", path: "/sync", error: { errcode: "NONONONONO" },
             });
             httpLookups.push({
-                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+                method: "GET", path: KEEP_ALIVE_PATH,
+                error: { errcode: "KEEPALIVE_FAIL" },
             });
 
             expectedStates.push(["PREPARED", null]);
@@ -425,9 +447,9 @@ describe("MatrixClient", function() {
 
         xit("should transition ERROR -> SYNCING after /sync if prev failed",
         function(done) {
-            var expectedStates = [];
+            const expectedStates = [];
             httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
+                method: "GET", path: "/sync", error: { errcode: "NONONONONO" },
             });
             httpLookups.push(SYNC_RESPONSE);
 
@@ -440,7 +462,7 @@ describe("MatrixClient", function() {
 
         it("should transition SYNCING -> SYNCING on subsequent /sync successes",
         function(done) {
-            var expectedStates = [];
+            const expectedStates = [];
             httpLookups.push(SYNC_RESPONSE);
             httpLookups.push(SYNC_RESPONSE);
 
@@ -452,16 +474,18 @@ describe("MatrixClient", function() {
         });
 
         it("should transition ERROR -> ERROR if keepalive keeps failing", function(done) {
-            accept_keepalives = false;
-            var expectedStates = [];
+            acceptKeepalives = false;
+            const expectedStates = [];
             httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
+                method: "GET", path: "/sync", error: { errcode: "NONONONONO" },
             });
             httpLookups.push({
-                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+                method: "GET", path: KEEP_ALIVE_PATH,
+                error: { errcode: "KEEPALIVE_FAIL" },
             });
             httpLookups.push({
-                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+                method: "GET", path: KEEP_ALIVE_PATH,
+                error: { errcode: "KEEPALIVE_FAIL" },
             });
 
             expectedStates.push(["PREPARED", null]);
@@ -475,7 +499,7 @@ describe("MatrixClient", function() {
     });
 
     describe("inviteByEmail", function() {
-        var roomId = "!foo:bar";
+        const roomId = "!foo:bar";
 
         it("should send an invite HTTP POST", function() {
             httpLookups = [{
@@ -485,17 +509,15 @@ describe("MatrixClient", function() {
                 expectBody: {
                     id_server: identityServerDomain,
                     medium: "email",
-                    address: "alice@gmail.com"
-                }
+                    address: "alice@gmail.com",
+                },
             }];
             client.inviteByEmail(roomId, "alice@gmail.com");
             expect(httpLookups.length).toEqual(0);
         });
-
     });
 
     describe("guest rooms", function() {
-
         it("should only do /sync calls (without filter/pushrules)", function(done) {
             httpLookups = []; // no /pushrules or /filter
             httpLookups.push({
@@ -504,7 +526,7 @@ describe("MatrixClient", function() {
                 data: SYNC_DATA,
                 thenCall: function() {
                     done();
-                }
+                },
             });
             client.setGuest(true);
             client.startClient();

spec/unit/pushprocessor.spec.js

@@ -1,33 +1,38 @@
 "use strict";
-var PushProcessor = require("../../lib/pushprocessor");
-var MatrixEvent = MatrixEvent;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const PushProcessor = require("../../lib/pushprocessor");
+const utils = require("../test-utils");
+
+import expect from 'expect';
 
 describe('NotificationService', function() {
-    var testUserId = "@ali:matrix.org";
-    var testDisplayName = "Alice M";
-    var testRoomId = "!fl1bb13:localhost";
+    const testUserId = "@ali:matrix.org";
+    const testDisplayName = "Alice M";
+    const testRoomId = "!fl1bb13:localhost";
 
-    var testEvent;
+    let testEvent;
 
-    var pushProcessor;
+    let pushProcessor;
 
     // These would be better if individual rules were configured in the tests themselves.
-    var matrixClient = {
+    const matrixClient = {
         getRoom: function() {
             return {
                 currentState: {
                     getMember: function() {
                         return {
-                            name: testDisplayName
+                            name: testDisplayName,
                         };
                     },
-                    members: {}
-                }
+                    getJoinedMemberCount: function() {
+                        return 0;
+                    },
+                    members: {},
+                },
             };
         },
         credentials: {
-            userId: testUserId
+            userId: testUserId,
         },
         pushRules: {
             "device": {},
@@ -38,91 +43,91 @@ describe('NotificationService', function() {
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "ali",
-                        "rule_id": ".m.rule.contains_user_name"
+                        "rule_id": ".m.rule.contains_user_name",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "coffee",
-                        "rule_id": "coffee"
+                        "rule_id": "coffee",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "foo*bar",
-                        "rule_id": "foobar"
+                        "rule_id": "foobar",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "p[io]ng",
-                        "rule_id": "pingpong"
+                        "rule_id": "pingpong",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "I ate [0-9] pies",
-                        "rule_id": "pies"
+                        "rule_id": "pies",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "enabled": true,
                         "pattern": "b[!ai]ke",
-                        "rule_id": "bakebike"
-                    }
+                        "rule_id": "bakebike",
+                    },
                 ],
                 "override": [
                     {
@@ -130,70 +135,70 @@ describe('NotificationService', function() {
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
+                                "value": "default",
                             },
                             {
-                                "set_tweak": "highlight"
-                            }
+                                "set_tweak": "highlight",
+                            },
                         ],
                         "conditions": [
                             {
-                                "kind": "contains_display_name"
-                            }
+                                "kind": "contains_display_name",
+                            },
                         ],
                         "enabled": true,
-                        "rule_id": ".m.rule.contains_display_name"
+                        "rule_id": ".m.rule.contains_display_name",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "sound",
-                                "value": "default"
-                            }
+                                "value": "default",
+                            },
                         ],
                         "conditions": [
                             {
                                 "is": "2",
-                                "kind": "room_member_count"
-                            }
+                                "kind": "room_member_count",
+                            },
                         ],
                         "enabled": true,
-                        "rule_id": ".m.rule.room_one_to_one"
-                    }
+                        "rule_id": ".m.rule.room_one_to_one",
+                    },
                 ],
                 "room": [],
                 "sender": [],
                 "underride": [
                     {
                         "actions": [
-                            "dont-notify"
+                            "dont-notify",
                         ],
                         "conditions": [
                             {
                                 "key": "content.msgtype",
                                 "kind": "event_match",
-                                "pattern": "m.notice"
-                            }
+                                "pattern": "m.notice",
+                            },
                         ],
                         "enabled": true,
-                        "rule_id": ".m.rule.suppress_notices"
+                        "rule_id": ".m.rule.suppress_notices",
                     },
                     {
                         "actions": [
                             "notify",
                             {
                                 "set_tweak": "highlight",
-                                "value": false
-                            }
+                                "value": false,
+                            },
                         ],
                         "conditions": [],
                         "enabled": true,
-                        "rule_id": ".m.rule.fallback"
-                    }
-                ]
-            }
-        }
+                        "rule_id": ".m.rule.fallback",
+                    },
+                ],
+            },
+        },
     };
 
     beforeEach(function() {
@@ -204,8 +209,8 @@ describe('NotificationService', function() {
             event: true,
             content: {
                 body: "",
-                msgtype: "m.text"
-            }
+                msgtype: "m.text",
+            },
         });
         pushProcessor = new PushProcessor(matrixClient);
     });
@@ -214,25 +219,25 @@ describe('NotificationService', function() {
 
     it('should bing on a user ID.', function() {
         testEvent.event.content.body = "Hello @ali:matrix.org, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on a partial user ID with an @.', function() {
         testEvent.event.content.body = "Hello @ali, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on a partial user ID without @.', function() {
         testEvent.event.content.body = "Hello ali, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on a case-insensitive user ID.', function() {
         testEvent.event.content.body = "Hello @AlI:matrix.org, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
@@ -240,13 +245,13 @@ describe('NotificationService', function() {
 
     it('should bing on a display name.', function() {
         testEvent.event.content.body = "Hello Alice M, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on a case-insensitive display name.', function() {
         testEvent.event.content.body = "Hello ALICE M, how are you?";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
@@ -254,25 +259,25 @@ describe('NotificationService', function() {
 
     it('should bing on a bing word.', function() {
         testEvent.event.content.body = "I really like coffee";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on case-insensitive bing words.', function() {
         testEvent.event.content.body = "Coffee is great";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on wildcard (.*) bing words.', function() {
         testEvent.event.content.body = "It was foomahbar I think.";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on character group ([abc]) bing words.', function() {
         testEvent.event.content.body = "Ping!";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        let actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
         testEvent.event.content.body = "Pong!";
         actions = pushProcessor.actionsForEvent(testEvent);
@@ -281,13 +286,13 @@ describe('NotificationService', function() {
 
     it('should bing on character range ([a-z]) bing words.', function() {
         testEvent.event.content.body = "I ate 6 pies";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
     });
 
     it('should bing on character negation ([!a]) bing words.', function() {
         testEvent.event.content.body = "boke";
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        let actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(true);
         testEvent.event.content.body = "bake";
         actions = pushProcessor.actionsForEvent(testEvent);
@@ -298,7 +303,7 @@ describe('NotificationService', function() {
 
     it('should gracefully handle bad input.', function() {
         testEvent.event.content.body = { "foo": "bar" };
-        var actions = pushProcessor.actionsForEvent(testEvent);
+        const actions = pushProcessor.actionsForEvent(testEvent);
         expect(actions.tweaks.highlight).toEqual(false);
     });
 });

spec/unit/realtime-callbacks.spec.js

@@ -1,97 +1,100 @@
 "use strict";
 
-var callbacks = require("../../lib/realtime-callbacks");
-var test_utils = require("../test-utils.js");
+import 'source-map-support/register';
+const callbacks = require("../../lib/realtime-callbacks");
+const testUtils = require("../test-utils.js");
+
+import expect from 'expect';
+import lolex from 'lolex';
 
 describe("realtime-callbacks", function() {
-    var clock = jasmine.Clock;
-    var fakeDate;
+    let clock;
 
     function tick(millis) {
-        // make sure we tick the fakedate first, otherwise nothing will happen!
-        fakeDate += millis;
         clock.tick(millis);
     }
 
     beforeEach(function() {
-        test_utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        clock.useMock();
-        fakeDate = Date.now();
-        callbacks.setNow(function() { return fakeDate; });
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+        clock = lolex.install();
+        const fakeDate = clock.Date;
+        callbacks.setNow(fakeDate.now.bind(fakeDate));
     });
 
     afterEach(function() {
         callbacks.setNow();
+        clock.uninstall();
     });
 
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
-            var callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 100);
 
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
             tick(100);
             expect(callback).toHaveBeenCalled();
         });
 
 
         it("should default to a zero timeout", function() {
-            var callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback);
 
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
             tick(0);
             expect(callback).toHaveBeenCalled();
         });
 
         it("should pass any parameters to the callback", function() {
-            var callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 0, "a", "b", "c");
             tick(0);
             expect(callback).toHaveBeenCalledWith("a", "b", "c");
         });
 
         it("should set 'this' to the global object", function() {
-            var callback = jasmine.createSpy();
-            callback.andCallFake(function() {
+            let passed = false;
+            const callback = function() {
                 expect(this).toBe(global); // eslint-disable-line no-invalid-this
-                expect(this.console).toBeDefined(); // eslint-disable-line no-invalid-this
-            });
+                expect(this.console).toBeTruthy(); // eslint-disable-line no-invalid-this
+                passed = true;
+            };
             callbacks.setTimeout(callback);
             tick(0);
-            expect(callback).toHaveBeenCalled();
+            expect(passed).toBe(true);
         });
 
         it("should handle timeouts of several seconds", function() {
-            var callback = jasmine.createSpy();
+            const callback = expect.createSpy();
             callbacks.setTimeout(callback, 2000);
 
-            expect(callback).not.toHaveBeenCalled();
-            for (var i = 0; i < 4; i++) {
+            expect(callback).toNotHaveBeenCalled();
+            for (let i = 0; i < 4; i++) {
                 tick(500);
             }
             expect(callback).toHaveBeenCalled();
         });
 
         it("should call multiple callbacks in the right order", function() {
-            var callback1 = jasmine.createSpy("callback1");
-            var callback2 = jasmine.createSpy("callback2");
-            var callback3 = jasmine.createSpy("callback3");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
+            const callback3 = expect.createSpy();
             callbacks.setTimeout(callback2, 200);
             callbacks.setTimeout(callback1, 100);
             callbacks.setTimeout(callback3, 300);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
-            expect(callback3).not.toHaveBeenCalled();
+            expect(callback3).toNotHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -99,81 +102,83 @@ describe("realtime-callbacks", function() {
         });
 
         it("should treat -ve timeouts the same as a zero timeout", function() {
-            var callback1 = jasmine.createSpy("callback1");
-            var callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
 
             // check that cb1 is called before cb2
-            callback1.andCallFake(function() {
-                expect(callback2).not.toHaveBeenCalled();
+            callback1.andCall(function() {
+                expect(callback2).toNotHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
             callbacks.setTimeout(callback2, -100);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should not get confused by chained calls", function() {
-            var callback2 = jasmine.createSpy("callback2");
-            var callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback2 = expect.createSpy();
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                 callbacks.setTimeout(callback2, 0);
-                expect(callback2).not.toHaveBeenCalled();
+                expect(callback2).toNotHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
+            // the fake timer won't actually run callbacks registered during
+            // one tick until the next tick.
+            tick(1);
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should be immune to exceptions", function() {
-            var callback1 = jasmine.createSpy("callback1");
-            callback1.andCallFake(function() {
+            const callback1 = expect.createSpy();
+            callback1.andCall(function() {
                 throw new Error("prepare to die");
             });
-            var callback2 = jasmine.createSpy("callback2");
+            const callback2 = expect.createSpy();
             callbacks.setTimeout(callback1, 0);
             callbacks.setTimeout(callback2, 0);
 
-            expect(callback1).not.toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback1).toNotHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
         });
-
     });
 
     describe("cancelTimeout", function() {
         it("should cancel a pending timeout", function() {
-            var callback = jasmine.createSpy();
-            var k = callbacks.setTimeout(callback);
+            const callback = expect.createSpy();
+            const k = callbacks.setTimeout(callback);
             callbacks.clearTimeout(k);
             tick(0);
-            expect(callback).not.toHaveBeenCalled();
+            expect(callback).toNotHaveBeenCalled();
         });
 
         it("should not affect sooner timeouts", function() {
-            var callback1 = jasmine.createSpy("callback1");
-            var callback2 = jasmine.createSpy("callback2");
+            const callback1 = expect.createSpy();
+            const callback2 = expect.createSpy();
 
             callbacks.setTimeout(callback1, 100);
-            var k = callbacks.setTimeout(callback2, 200);
+            const k = callbacks.setTimeout(callback2, 200);
             callbacks.clearTimeout(k);
 
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
 
             tick(150);
-            expect(callback2).not.toHaveBeenCalled();
+            expect(callback2).toNotHaveBeenCalled();
         });
     });
 });

spec/unit/room-member.spec.js

@@ -1,14 +1,17 @@
 "use strict";
-var sdk = require("../..");
-var RoomMember = sdk.RoomMember;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const RoomMember = sdk.RoomMember;
+const utils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("RoomMember", function() {
-    var roomId = "!foo:bar";
-    var userA = "@alice:bar";
-    var userB = "@bertha:bar";
-    var userC = "@clarissa:bar";
-    var member;
+    const roomId = "!foo:bar";
+    const userA = "@alice:bar";
+    const userB = "@bertha:bar";
+    const userC = "@clarissa:bar";
+    let member;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -16,7 +19,7 @@ describe("RoomMember", function() {
     });
 
     describe("getAvatarUrl", function() {
-        var hsUrl = "https://my.home.server";
+        const hsUrl = "https://my.home.server";
 
         it("should return the URL from m.room.member preferentially", function() {
             member.events.member = utils.mkEvent({
@@ -27,31 +30,31 @@ describe("RoomMember", function() {
                 user: userA,
                 content: {
                     membership: "join",
-                    avatar_url: "mxc://flibble/wibble"
-                }
+                    avatar_url: "mxc://flibble/wibble",
+                },
             });
-            var url = member.getAvatarUrl(hsUrl);
+            const url = member.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
+            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
         });
 
         it("should return an identicon HTTP URL if allowDefault was set and there " +
         "was no m.room.member event", function() {
-            var url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
+            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
             expect(url.indexOf("http")).toEqual(0); // don't care about form
         });
 
         it("should return nothing if there is no m.room.member and allowDefault=false",
         function() {
-            var url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);
+            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);
             expect(url).toEqual(null);
         });
     });
 
     describe("setPowerLevelEvent", function() {
         it("should set 'powerLevel' and 'powerLevelNorm'.", function() {
-            var event = utils.mkEvent({
+            const event = utils.mkEvent({
                 type: "m.room.power_levels",
                 room: roomId,
                 user: userA,
@@ -59,16 +62,16 @@ describe("RoomMember", function() {
                     users_default: 20,
                     users: {
                         "@bertha:bar": 200,
-                        "@invalid:user": 10  // shouldn't barf on this.
-                    }
+                        "@invalid:user": 10,  // shouldn't barf on this.
+                    },
                 },
-                event: true
+                event: true,
             });
             member.setPowerLevelEvent(event);
             expect(member.powerLevel).toEqual(20);
             expect(member.powerLevelNorm).toEqual(10);
 
-            var memberB = new RoomMember(roomId, userB);
+            const memberB = new RoomMember(roomId, userB);
             memberB.setPowerLevelEvent(event);
             expect(memberB.powerLevel).toEqual(200);
             expect(memberB.powerLevelNorm).toEqual(100);
@@ -76,7 +79,7 @@ describe("RoomMember", function() {
 
         it("should emit 'RoomMember.powerLevel' if the power level changes.",
         function() {
-            var event = utils.mkEvent({
+            const event = utils.mkEvent({
                 type: "m.room.power_levels",
                 room: roomId,
                 user: userA,
@@ -84,12 +87,12 @@ describe("RoomMember", function() {
                     users_default: 20,
                     users: {
                         "@bertha:bar": 200,
-                        "@invalid:user": 10  // shouldn't barf on this.
-                    }
+                        "@invalid:user": 10,  // shouldn't barf on this.
+                    },
                 },
-                event: true
+                event: true,
             });
-            var emitCount = 0;
+            let emitCount = 0;
 
             member.on("RoomMember.powerLevel", function(emitEvent, emitMember) {
                 emitCount += 1;
@@ -105,7 +108,7 @@ describe("RoomMember", function() {
 
         it("should honour power levels of zero.",
         function() {
-            var event = utils.mkEvent({
+            const event = utils.mkEvent({
                 type: "m.room.power_levels",
                 room: roomId,
                 user: userA,
@@ -113,11 +116,11 @@ describe("RoomMember", function() {
                     users_default: 20,
                     users: {
                         "@alice:bar": 0,
-                    }
+                    },
                 },
-                event: true
+                event: true,
             });
-            var emitCount = 0;
+            let emitCount = 0;
 
             // set the power level to something other than zero or we
             // won't get an event
@@ -138,21 +141,21 @@ describe("RoomMember", function() {
     describe("setTypingEvent", function() {
         it("should set 'typing'", function() {
             member.typing = false;
-            var memberB = new RoomMember(roomId, userB);
+            const memberB = new RoomMember(roomId, userB);
             memberB.typing = true;
-            var memberC = new RoomMember(roomId, userC);
+            const memberC = new RoomMember(roomId, userC);
             memberC.typing = true;
 
-            var event = utils.mkEvent({
+            const event = utils.mkEvent({
                 type: "m.typing",
                 user: userA,
                 room: roomId,
                 content: {
                     user_ids: [
-                        userA, userC
-                    ]
+                        userA, userC,
+                    ],
                 },
-                event: true
+                event: true,
             });
             member.setTypingEvent(event);
             memberB.setTypingEvent(event);
@@ -165,17 +168,17 @@ describe("RoomMember", function() {
 
         it("should emit 'RoomMember.typing' if the typing state changes",
         function() {
-            var event = utils.mkEvent({
+            const event = utils.mkEvent({
                 type: "m.typing",
                 room: roomId,
                 content: {
                     user_ids: [
-                        userA, userC
-                    ]
+                        userA, userC,
+                    ],
                 },
-                event: true
+                event: true,
             });
-            var emitCount = 0;
+            let emitCount = 0;
             member.on("RoomMember.typing", function(ev, mem) {
                 expect(mem).toEqual(member);
                 expect(ev).toEqual(event);
@@ -189,21 +192,30 @@ describe("RoomMember", function() {
         });
     });
 
+    describe("isOutOfBand", function() {
+        it("should be set by markOutOfBand", function() {
+            const member = new RoomMember();
+            expect(member.isOutOfBand()).toEqual(false);
+            member.markOutOfBand();
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+    });
+
     describe("setMembershipEvent", function() {
-        var joinEvent = utils.mkMembership({
+        const joinEvent = utils.mkMembership({
             event: true,
             mship: "join",
             user: userA,
             room: roomId,
-            name: "Alice"
+            name: "Alice",
         });
 
-        var inviteEvent = utils.mkMembership({
+        const inviteEvent = utils.mkMembership({
             event: true,
             mship: "invite",
             user: userB,
             skey: userA,
-            room: roomId
+            room: roomId,
         });
 
         it("should set 'membership' and assign the event to 'events.member'.",
@@ -218,36 +230,38 @@ describe("RoomMember", function() {
 
         it("should set 'name' based on user_id, displayname and room state",
         function() {
-            var roomState = {
+            const roomState = {
                 getStateEvents: function(type) {
-                    if (type !== "m.room.member") { return []; }
+                    if (type !== "m.room.member") {
+                        return [];
+                    }
                     return [
                         utils.mkMembership({
                             event: true, mship: "join", room: roomId,
-                            user: userB
+                            user: userB,
                         }),
                         utils.mkMembership({
                             event: true, mship: "join", room: roomId,
-                            user: userC, name: "Alice"
+                            user: userC, name: "Alice",
                         }),
-                        joinEvent
+                        joinEvent,
                     ];
                 },
                 getUserIdsWithDisplayName: function(displayName) {
                     return [userA, userC];
-                }
+                },
             };
             expect(member.name).toEqual(userA); // default = user_id
             member.setMembershipEvent(joinEvent);
             expect(member.name).toEqual("Alice"); // prefer displayname
             member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).not.toEqual("Alice"); // it should disambig.
+            expect(member.name).toNotEqual("Alice"); // it should disambig.
             // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).not.toEqual(-1);
+            expect(member.name.indexOf(userA)).toNotEqual(-1);
         });
 
         it("should emit 'RoomMember.membership' if the membership changes", function() {
-            var emitCount = 0;
+            let emitCount = 0;
             member.on("RoomMember.membership", function(ev, mem) {
                 emitCount += 1;
                 expect(mem).toEqual(member);
@@ -260,7 +274,7 @@ describe("RoomMember", function() {
         });
 
         it("should emit 'RoomMember.name' if the name changes", function() {
-            var emitCount = 0;
+            let emitCount = 0;
             member.on("RoomMember.name", function(ev, mem) {
                 emitCount += 1;
                 expect(mem).toEqual(member);
@@ -271,8 +285,5 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent); // no-op
             expect(emitCount).toEqual(1);
         });
-
-
     });
-
 });

spec/unit/room-state.spec.js

@@ -1,35 +1,41 @@
 "use strict";
-var sdk = require("../..");
-var RoomState = sdk.RoomState;
-var RoomMember = sdk.RoomMember;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const RoomState = sdk.RoomState;
+const RoomMember = sdk.RoomMember;
+const utils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("RoomState", function() {
-    var roomId = "!foo:bar";
-    var userA = "@alice:bar";
-    var userB = "@bob:bar";
-    var state;
+    const roomId = "!foo:bar";
+    const userA = "@alice:bar";
+    const userB = "@bob:bar";
+    const userC = "@cleo:bar";
+    const userLazy = "@lazy:bar";
+
+    let state;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
         state = new RoomState(roomId);
         state.setStateEvents([
             utils.mkMembership({  // userA joined
-                event: true, mship: "join", user: userA, room: roomId
+                event: true, mship: "join", user: userA, room: roomId,
             }),
             utils.mkMembership({  // userB joined
-                event: true, mship: "join", user: userB, room: roomId
+                event: true, mship: "join", user: userB, room: roomId,
             }),
             utils.mkEvent({  // Room name is "Room name goes here"
                 type: "m.room.name", user: userA, room: roomId, event: true, content: {
-                    name: "Room name goes here"
-                }
+                    name: "Room name goes here",
+                },
             }),
             utils.mkEvent({  // Room creation
                 type: "m.room.create", user: userA, room: roomId, event: true, content: {
-                    creator: userA
-                }
-            })
+                    creator: userA,
+                },
+            }),
         ]);
     });
 
@@ -40,11 +46,11 @@ describe("RoomState", function() {
         });
 
         it("should return a member for each m.room.member event", function() {
-            var members = state.getMembers();
+            const members = state.getMembers();
             expect(members.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(members[0].userId)).not.toEqual(-1);
-            expect([userA, userB].indexOf(members[1].userId)).not.toEqual(-1);
+            expect([userA, userB].indexOf(members[0].userId)).toNotEqual(-1);
+            expect([userA, userB].indexOf(members[1].userId)).toNotEqual(-1);
         });
     });
 
@@ -54,19 +60,19 @@ describe("RoomState", function() {
         });
 
         it("should return a member if they exist", function() {
-            expect(state.getMember(userB)).toBeDefined();
+            expect(state.getMember(userB)).toBeTruthy();
         });
 
         it("should return a member which changes as state changes", function() {
-            var member = state.getMember(userB);
+            const member = state.getMember(userB);
             expect(member.membership).toEqual("join");
             expect(member.name).toEqual(userB);
 
             state.setStateEvents([
                 utils.mkMembership({
                     room: roomId, user: userB, mship: "leave", event: true,
-                    name: "BobGone"
-                })
+                    name: "BobGone",
+                }),
             ]);
 
             expect(member.membership).toEqual("leave");
@@ -75,20 +81,20 @@ describe("RoomState", function() {
     });
 
     describe("getSentinelMember", function() {
-        it("should return null if there is no member", function() {
-            expect(state.getSentinelMember("@no-one:here")).toEqual(null);
+        it("should return a member with the user id as name", function() {
+            expect(state.getSentinelMember("@no-one:here").name).toEqual("@no-one:here");
         });
 
         it("should return a member which doesn't change when the state is updated",
         function() {
-            var preLeaveUser = state.getSentinelMember(userA);
+            const preLeaveUser = state.getSentinelMember(userA);
             state.setStateEvents([
                 utils.mkMembership({
                     room: roomId, user: userA, mship: "leave", event: true,
-                    name: "AliceIsGone"
-                })
+                    name: "AliceIsGone",
+                }),
             ]);
-            var postLeaveUser = state.getSentinelMember(userA);
+            const postLeaveUser = state.getSentinelMember(userA);
 
             expect(preLeaveUser.membership).toEqual("join");
             expect(preLeaveUser.name).toEqual(userA);
@@ -111,33 +117,33 @@ describe("RoomState", function() {
 
         it("should return a list of matching events if no state_key was specified",
         function() {
-            var events = state.getStateEvents("m.room.member");
+            const events = state.getStateEvents("m.room.member");
             expect(events.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(events[0].getStateKey())).not.toEqual(-1);
-            expect([userA, userB].indexOf(events[1].getStateKey())).not.toEqual(-1);
+            expect([userA, userB].indexOf(events[0].getStateKey())).toNotEqual(-1);
+            expect([userA, userB].indexOf(events[1].getStateKey())).toNotEqual(-1);
         });
 
         it("should return a single MatrixEvent if a state_key was specified",
         function() {
-            var event = state.getStateEvents("m.room.member", userA);
+            const event = state.getStateEvents("m.room.member", userA);
             expect(event.getContent()).toEqual({
-                membership: "join"
+                membership: "join",
             });
         });
     });
 
     describe("setStateEvents", function() {
         it("should emit 'RoomState.members' for each m.room.member event", function() {
-            var memberEvents = [
+            const memberEvents = [
                 utils.mkMembership({
-                    user: "@cleo:bar", mship: "invite", room: roomId, event: true
+                    user: "@cleo:bar", mship: "invite", room: roomId, event: true,
                 }),
                 utils.mkMembership({
-                    user: "@daisy:bar", mship: "join", room: roomId, event: true
-                })
+                    user: "@daisy:bar", mship: "join", room: roomId, event: true,
+                }),
             ];
-            var emitCount = 0;
+            let emitCount = 0;
             state.on("RoomState.members", function(ev, st, mem) {
                 expect(ev).toEqual(memberEvents[emitCount]);
                 expect(st).toEqual(state);
@@ -149,16 +155,17 @@ describe("RoomState", function() {
         });
 
         it("should emit 'RoomState.newMember' for each new member added", function() {
-            var memberEvents = [
+            const memberEvents = [
                 utils.mkMembership({
-                    user: "@cleo:bar", mship: "invite", room: roomId, event: true
+                    user: "@cleo:bar", mship: "invite", room: roomId, event: true,
                 }),
                 utils.mkMembership({
-                    user: "@daisy:bar", mship: "join", room: roomId, event: true
-                })
+                    user: "@daisy:bar", mship: "join", room: roomId, event: true,
+                }),
             ];
-            var emitCount = 0;
+            let emitCount = 0;
             state.on("RoomState.newMember", function(ev, st, mem) {
+                expect(state.getMember(mem.userId)).toEqual(mem);
                 expect(mem.userId).toEqual(memberEvents[emitCount].getSender());
                 expect(mem.membership).toBeFalsy();  // not defined yet
                 emitCount += 1;
@@ -168,21 +175,21 @@ describe("RoomState", function() {
         });
 
         it("should emit 'RoomState.events' for each state event", function() {
-            var events = [
+            const events = [
                 utils.mkMembership({
-                    user: "@cleo:bar", mship: "invite", room: roomId, event: true
+                    user: "@cleo:bar", mship: "invite", room: roomId, event: true,
                 }),
                 utils.mkEvent({
                     user: userB, room: roomId, type: "m.room.topic", event: true,
                     content: {
-                        topic: "boo!"
-                    }
+                        topic: "boo!",
+                    },
                 }),
                 utils.mkMessage({  // Not a state event
-                    user: userA, room: roomId, event: true
-                })
+                    user: userA, room: roomId, event: true,
+                }),
             ];
-            var emitCount = 0;
+            let emitCount = 0;
             state.on("RoomState.events", function(ev, st) {
                 expect(ev).toEqual(events[emitCount]);
                 expect(st).toEqual(state);
@@ -198,39 +205,38 @@ describe("RoomState", function() {
             state.members[userA] = utils.mock(RoomMember);
             state.members[userB] = utils.mock(RoomMember);
 
-            var powerLevelEvent = utils.mkEvent({
-                type: "m.room.power_levels", room: roomId, user: userA, event: true,
-                content: {
-                    users_default: 10,
-                    state_default: 50,
-                    events_default: 25
-                }
-            });
-
-            state.setStateEvents([powerLevelEvent]);
-
-            expect(state.members[userA].setPowerLevelEvent).toHaveBeenCalledWith(
-                powerLevelEvent
-            );
-            expect(state.members[userB].setPowerLevelEvent).toHaveBeenCalledWith(
-                powerLevelEvent
-            );
-        });
-
-        it("should call setPowerLevelEvent on a new RoomMember if power levels exist",
-        function() {
-            var userC = "@cleo:bar";
-            var memberEvent = utils.mkMembership({
-                mship: "join", user: userC, room: roomId, event: true
-            });
-            var powerLevelEvent = utils.mkEvent({
+            const powerLevelEvent = utils.mkEvent({
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     users_default: 10,
                     state_default: 50,
                     events_default: 25,
-                    users: {}
-                }
+                },
+            });
+
+            state.setStateEvents([powerLevelEvent]);
+
+            expect(state.members[userA].setPowerLevelEvent).toHaveBeenCalledWith(
+                powerLevelEvent,
+            );
+            expect(state.members[userB].setPowerLevelEvent).toHaveBeenCalledWith(
+                powerLevelEvent,
+            );
+        });
+
+        it("should call setPowerLevelEvent on a new RoomMember if power levels exist",
+        function() {
+            const memberEvent = utils.mkMembership({
+                mship: "join", user: userC, room: roomId, event: true,
+            });
+            const powerLevelEvent = utils.mkEvent({
+                type: "m.room.power_levels", room: roomId, user: userA, event: true,
+                content: {
+                    users_default: 10,
+                    state_default: 50,
+                    events_default: 25,
+                    users: {},
+                },
             });
 
             state.setStateEvents([powerLevelEvent]);
@@ -238,7 +244,7 @@ describe("RoomState", function() {
 
             // TODO: We do this because we don't DI the RoomMember constructor
             // so we can't inject a mock :/ so we have to infer.
-            expect(state.members[userC]).toBeDefined();
+            expect(state.members[userC]).toBeTruthy();
             expect(state.members[userC].powerLevel).toEqual(10);
         });
 
@@ -247,24 +253,132 @@ describe("RoomState", function() {
             state.members[userA] = utils.mock(RoomMember);
             state.members[userB] = utils.mock(RoomMember);
 
-            var memberEvent = utils.mkMembership({
-                user: userB, mship: "leave", room: roomId, event: true
+            const memberEvent = utils.mkMembership({
+                user: userB, mship: "leave", room: roomId, event: true,
             });
             state.setStateEvents([memberEvent]);
 
-            expect(state.members[userA].setMembershipEvent).not.toHaveBeenCalled();
+            expect(state.members[userA].setMembershipEvent).toNotHaveBeenCalled();
             expect(state.members[userB].setMembershipEvent).toHaveBeenCalledWith(
-                memberEvent, state
+                memberEvent, state,
             );
         });
     });
 
+    describe("setOutOfBandMembers", function() {
+        it("should add a new member", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const member = state.getMember(userLazy);
+            expect(member.userId).toEqual(userLazy);
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+
+        it("should have no effect when not in correct status", function() {
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should emit newMember when adding a member", function() {
+            const userLazy = "@oob:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.newMember', (_, __, member) => {
+                expect(member.userId).toEqual(userLazy);
+                eventReceived = true;
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+
+        it("should never overwrite existing members", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const memberA = state.getMember(userA);
+            expect(memberA.events.member.getId()).toNotEqual(oobMemberEvent.getId());
+            expect(memberA.isOutOfBand()).toEqual(false);
+        });
+
+        it("should emit members when updating a member", function() {
+            const doesntExistYetUserId = "@doesntexistyet:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: doesntExistYetUserId, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.members', (_, __, member) => {
+                expect(member.userId).toEqual(doesntExistYetUserId);
+                eventReceived = true;
+            });
+
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+    });
+
+    describe("clone", function() {
+        it("should contain same information as original", function() {
+            // include OOB members in copy
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            const copy = state.clone();
+            // check individual members
+            [userA, userB, userLazy].forEach((userId) => {
+                const member = state.getMember(userId);
+                const memberCopy = copy.getMember(userId);
+                expect(member.name).toEqual(memberCopy.name);
+                expect(member.isOutOfBand()).toEqual(memberCopy.isOutOfBand());
+            });
+            // check member keys
+            expect(Object.keys(state.members)).toEqual(Object.keys(copy.members));
+            // check join count
+            expect(state.getJoinedMemberCount()).toEqual(copy.getJoinedMemberCount());
+        });
+
+        it("should mark old copy as not waiting for out of band anymore", function() {
+            state.markOutOfBandMembersStarted();
+            const copy = state.clone();
+            copy.setOutOfBandMembers([utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            })]);
+            // should have no effect as it should be marked in status finished just like copy
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should return copy independent of original", function() {
+            const copy = state.clone();
+            copy.setStateEvents([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+
+            expect(state.getMember(userLazy)).toBeFalsy();
+            expect(state.getJoinedMemberCount()).toEqual(2);
+            expect(copy.getJoinedMemberCount()).toEqual(3);
+        });
+    });
+
     describe("setTypingEvent", function() {
         it("should call setTypingEvent on each RoomMember", function() {
-            var typingEvent = utils.mkEvent({
+            const typingEvent = utils.mkEvent({
                 type: "m.typing", room: roomId, event: true, content: {
-                    user_ids: [userA]
-                }
+                    user_ids: [userA],
+                },
             });
             // mock up the room members
             state.members[userA] = utils.mock(RoomMember);
@@ -272,22 +386,15 @@ describe("RoomState", function() {
             state.setTypingEvent(typingEvent);
 
             expect(state.members[userA].setTypingEvent).toHaveBeenCalledWith(
-                typingEvent
+                typingEvent,
             );
             expect(state.members[userB].setTypingEvent).toHaveBeenCalledWith(
-                typingEvent
+                typingEvent,
             );
         });
     });
 
     describe("maySendStateEvent", function() {
-        it("should say non-joined members may not send state",
-        function() {
-            expect(state.maySendStateEvent(
-                'm.room.name', "@nobody:nowhere"
-            )).toEqual(false);
-        });
-
         it("should say any member may send state with no power level event",
         function() {
             expect(state.maySendStateEvent('m.room.name', userA)).toEqual(true);
@@ -296,15 +403,15 @@ describe("RoomState", function() {
         it("should say members with power >=50 may send state with power level event " +
         "but no state default",
         function() {
-            var powerLevelEvent = {
+            const powerLevelEvent = {
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     users_default: 10,
                     // state_default: 50, "intentionally left blank"
                     events_default: 25,
                     users: {
-                    }
-                }
+                    },
+                },
             };
             powerLevelEvent.content.users[userA] = 50;
 
@@ -316,15 +423,15 @@ describe("RoomState", function() {
 
         it("should obey state_default",
         function() {
-            var powerLevelEvent = {
+            const powerLevelEvent = {
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     users_default: 10,
                     state_default: 30,
                     events_default: 25,
                     users: {
-                    }
-                }
+                    },
+                },
             };
             powerLevelEvent.content.users[userA] = 30;
             powerLevelEvent.content.users[userB] = 29;
@@ -337,18 +444,18 @@ describe("RoomState", function() {
 
         it("should honour explicit event power levels in the power_levels event",
         function() {
-            var powerLevelEvent = {
+            const powerLevelEvent = {
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     events: {
-                        "m.room.other_thing": 76
+                        "m.room.other_thing": 76,
                     },
                     users_default: 10,
                     state_default: 50,
                     events_default: 25,
                     users: {
-                    }
-                }
+                    },
+                },
             };
             powerLevelEvent.content.users[userA] = 80;
             powerLevelEvent.content.users[userB] = 50;
@@ -363,15 +470,117 @@ describe("RoomState", function() {
         });
     });
 
-    describe("maySendEvent", function() {
-        it("should say non-joined members may not send events",
-        function() {
-            expect(state.maySendEvent(
-                'm.room.message', "@nobody:nowhere"
-            )).toEqual(false);
-            expect(state.maySendMessage("@nobody:nowhere")).toEqual(false);
+    describe("getJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
         });
 
+        it("should update after adding joined member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("getInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should update after adding invited member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("setJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setJoinedMemberCount(100);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            state.setJoinedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("setInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setInvitedMemberCount(100);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            state.setInvitedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("maySendEvent", function() {
         it("should say any member may send events with no power level event",
         function() {
             expect(state.maySendEvent('m.room.message', userA)).toEqual(true);
@@ -380,15 +589,15 @@ describe("RoomState", function() {
 
         it("should obey events_default",
         function() {
-            var powerLevelEvent = {
+            const powerLevelEvent = {
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     users_default: 10,
                     state_default: 30,
                     events_default: 25,
                     users: {
-                    }
-                }
+                    },
+                },
             };
             powerLevelEvent.content.users[userA] = 26;
             powerLevelEvent.content.users[userB] = 24;
@@ -404,18 +613,18 @@ describe("RoomState", function() {
 
         it("should honour explicit event power levels in the power_levels event",
         function() {
-            var powerLevelEvent = {
+            const powerLevelEvent = {
                 type: "m.room.power_levels", room: roomId, user: userA, event: true,
                 content: {
                     events: {
-                        "m.room.other_thing": 33
+                        "m.room.other_thing": 33,
                     },
                     users_default: 10,
                     state_default: 50,
                     events_default: 25,
                     users: {
-                    }
-                }
+                    },
+                },
             };
             powerLevelEvent.content.users[userA] = 40;
             powerLevelEvent.content.users[userB] = 30;

spec/unit/scheduler.spec.js

@@ -1,27 +1,33 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */
 
-var q = require("q");
-var sdk = require("../..");
-var MatrixScheduler = sdk.MatrixScheduler;
-var MatrixError = sdk.MatrixError;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const MatrixScheduler = sdk.MatrixScheduler;
+const MatrixError = sdk.MatrixError;
+const utils = require("../test-utils");
+
+import expect from 'expect';
+import lolex from 'lolex';
 
 describe("MatrixScheduler", function() {
-    var scheduler;
-    var retryFn, queueFn;
-    var defer;
-    var roomId = "!foo:bar";
-    var eventA = utils.mkMessage({
-        user: "@alice:bar", room: roomId, event: true
+    let clock;
+    let scheduler;
+    let retryFn;
+    let queueFn;
+    let defer;
+    const roomId = "!foo:bar";
+    const eventA = utils.mkMessage({
+        user: "@alice:bar", room: roomId, event: true,
     });
-    var eventB = utils.mkMessage({
-        user: "@alice:bar", room: roomId, event: true
+    const eventB = utils.mkMessage({
+        user: "@alice:bar", room: roomId, event: true,
     });
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        jasmine.Clock.useMock();
+        clock = lolex.install();
         scheduler = new MatrixScheduler(function(ev, attempts, err) {
             if (retryFn) {
                 return retryFn(ev, attempts, err);
@@ -35,7 +41,11 @@ describe("MatrixScheduler", function() {
         });
         retryFn = null;
         queueFn = null;
-        defer = q.defer();
+        defer = Promise.defer();
+    });
+
+    afterEach(function() {
+        clock.uninstall();
     });
 
     it("should process events in a queue in a FIFO manner", function(done) {
@@ -45,15 +55,14 @@ describe("MatrixScheduler", function() {
         queueFn = function() {
             return "one_big_queue";
         };
-        var deferA = q.defer();
-        var deferB = q.defer();
-        var resolvedA = false;
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
+        let resolvedA = false;
         scheduler.setProcessFunction(function(event) {
             if (resolvedA) {
                 expect(event).toEqual(eventB);
                 return deferB.promise;
-            }
-            else {
+            } else {
                 expect(event).toEqual(eventA);
                 return deferA.promise;
             }
@@ -70,24 +79,25 @@ describe("MatrixScheduler", function() {
 
     it("should invoke the retryFn on failure and wait the amount of time specified",
     function(done) {
-        var waitTimeMs = 1500;
-        var retryDefer = q.defer();
+        const waitTimeMs = 1500;
+        const retryDefer = Promise.defer();
         retryFn = function() {
             retryDefer.resolve();
             return waitTimeMs;
         };
-        queueFn = function() { return "yep"; };
+        queueFn = function() {
+            return "yep";
+        };
 
-        var procCount = 0;
+        let procCount = 0;
         scheduler.setProcessFunction(function(ev) {
             procCount += 1;
             if (procCount === 1) {
                 expect(ev).toEqual(eventA);
                 return defer.promise;
-            }
-            else if (procCount === 2) {
+            } else if (procCount === 2) {
                 // don't care about this defer
-                return q.defer().promise;
+                return Promise.defer().promise;
             }
             expect(procCount).toBeLessThan(3);
         });
@@ -97,7 +107,7 @@ describe("MatrixScheduler", function() {
         defer.reject({});
         retryDefer.promise.done(function() {
             expect(procCount).toEqual(1);
-            jasmine.Clock.tick(waitTimeMs);
+            clock.tick(waitTimeMs);
             expect(procCount).toEqual(2);
             done();
         });
@@ -111,25 +121,26 @@ describe("MatrixScheduler", function() {
         retryFn = function() {
             return -1;
         };
-        queueFn = function() { return "yep"; };
+        queueFn = function() {
+ return "yep";
+};
 
-        var deferA = q.defer();
-        var deferB = q.defer();
-        var procCount = 0;
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
+        let procCount = 0;
         scheduler.setProcessFunction(function(ev) {
             procCount += 1;
             if (procCount === 1) {
                 expect(ev).toEqual(eventA);
                 return deferA.promise;
-            }
-            else if (procCount === 2) {
+            } else if (procCount === 2) {
                 expect(ev).toEqual(eventB);
                 return deferB.promise;
             }
             expect(procCount).toBeLessThan(3);
         });
 
-        var globalA = scheduler.queueEvent(eventA);
+        const globalA = scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
 
         expect(procCount).toEqual(1);
@@ -147,10 +158,10 @@ describe("MatrixScheduler", function() {
         // Expect to have processFn invoked for A&B.
         // Resolve A.
         // Expect to have processFn invoked for D.
-        var eventC = utils.mkMessage({user: "@a:bar", room: roomId, event: true});
-        var eventD = utils.mkMessage({user: "@b:bar", room: roomId, event: true});
+        const eventC = utils.mkMessage({user: "@a:bar", room: roomId, event: true});
+        const eventD = utils.mkMessage({user: "@b:bar", room: roomId, event: true});
 
-        var buckets = {};
+        const buckets = {};
         buckets[eventA.getId()] = "queue_A";
         buckets[eventD.getId()] = "queue_A";
         buckets[eventB.getId()] = "queue_B";
@@ -163,12 +174,12 @@ describe("MatrixScheduler", function() {
             return buckets[event.getId()];
         };
 
-        var expectOrder = [
-            eventA.getId(), eventB.getId(), eventD.getId()
+        const expectOrder = [
+            eventA.getId(), eventB.getId(), eventD.getId(),
         ];
-        var deferA = q.defer();
+        const deferA = Promise.defer();
         scheduler.setProcessFunction(function(event) {
-            var id = expectOrder.shift();
+            const id = expectOrder.shift();
             expect(id).toEqual(event.getId());
             if (expectOrder.length === 0) {
                 done();
@@ -184,7 +195,7 @@ describe("MatrixScheduler", function() {
         setTimeout(function() {
             deferA.resolve({});
         }, 1000);
-        jasmine.Clock.tick(1000);
+        clock.tick(1000);
     });
 
     describe("queueEvent", function() {
@@ -199,9 +210,9 @@ describe("MatrixScheduler", function() {
             queueFn = function() {
                 return "yep";
             };
-            var prom = scheduler.queueEvent(eventA);
-            expect(prom).toBeDefined();
-            expect(prom.then).toBeDefined();
+            const prom = scheduler.queueEvent(eventA);
+            expect(prom).toBeTruthy();
+            expect(prom.then).toBeTruthy();
         });
     });
 
@@ -210,14 +221,14 @@ describe("MatrixScheduler", function() {
             queueFn = function() {
                 return null;
             };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
         });
 
         it("should return null if the mapped queue doesn't exist", function() {
             queueFn = function() {
                 return "yep";
             };
-            expect(scheduler.getQueueForEvent(eventA)).toBeNull();
+            expect(scheduler.getQueueForEvent(eventA)).toBe(null);
         });
 
         it("should return a list of events in the queue and modifications to" +
@@ -227,15 +238,15 @@ describe("MatrixScheduler", function() {
             };
             scheduler.queueEvent(eventA);
             scheduler.queueEvent(eventB);
-            var queue = scheduler.getQueueForEvent(eventA);
+            const queue = scheduler.getQueueForEvent(eventA);
             expect(queue.length).toEqual(2);
             expect(queue).toEqual([eventA, eventB]);
             // modify the queue
-            var eventC = utils.mkMessage(
-                {user: "@a:bar", room: roomId, event: true}
+            const eventC = utils.mkMessage(
+                {user: "@a:bar", room: roomId, event: true},
             );
             queue.push(eventC);
-            var queueAgain = scheduler.getQueueForEvent(eventA);
+            const queueAgain = scheduler.getQueueForEvent(eventA);
             expect(queueAgain.length).toEqual(2);
         });
 
@@ -246,9 +257,9 @@ describe("MatrixScheduler", function() {
             };
             scheduler.queueEvent(eventA);
             scheduler.queueEvent(eventB);
-            var queue = scheduler.getQueueForEvent(eventA);
+            const queue = scheduler.getQueueForEvent(eventA);
             queue[1].event.content.body = "foo";
-            var queueAgain = scheduler.getQueueForEvent(eventA);
+            const queueAgain = scheduler.getQueueForEvent(eventA);
             expect(queueAgain[1].event.content.body).toEqual("foo");
         });
     });
@@ -282,7 +293,7 @@ describe("MatrixScheduler", function() {
             queueFn = function() {
                 return "yep";
             };
-            var procCount = 0;
+            let procCount = 0;
             scheduler.queueEvent(eventA);
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
@@ -296,7 +307,7 @@ describe("MatrixScheduler", function() {
             queueFn = function() {
                 return "yep";
             };
-            var procCount = 0;
+            let procCount = 0;
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
                 return defer.promise;
@@ -310,41 +321,41 @@ describe("MatrixScheduler", function() {
             expect(MatrixScheduler.QUEUE_MESSAGES(eventA)).toEqual("message");
             expect(MatrixScheduler.QUEUE_MESSAGES(
                 utils.mkMembership({
-                    user: "@alice:bar", room: roomId, mship: "join", event: true
-                })
+                    user: "@alice:bar", room: roomId, mship: "join", event: true,
+                }),
             )).toEqual(null);
         });
     });
 
     describe("RETRY_BACKOFF_RATELIMIT", function() {
         it("should wait at least the time given on M_LIMIT_EXCEEDED", function() {
-            var res = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
+            const res = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
                 eventA, 1, new MatrixError({
-                    errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 5000
-                })
+                    errcode: "M_LIMIT_EXCEEDED", retry_after_ms: 5000,
+                }),
             );
             expect(res >= 500).toBe(true, "Didn't wait long enough.");
         });
 
         it("should give up after 5 attempts", function() {
-            var res = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 5, {}
+            const res = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
+                eventA, 5, {},
             );
             expect(res).toBe(-1, "Didn't give up.");
         });
 
         it("should do exponential backoff", function() {
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 1, {}
+                eventA, 1, {},
             )).toEqual(2000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 2, {}
+                eventA, 2, {},
             )).toEqual(4000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 3, {}
+                eventA, 3, {},
             )).toEqual(8000);
             expect(MatrixScheduler.RETRY_BACKOFF_RATELIMIT(
-                eventA, 4, {}
+                eventA, 4, {},
             )).toEqual(16000);
         });
     });

spec/unit/sync-accumulator.spec.js

@@ -0,0 +1,413 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+import 'source-map-support/register';
+import utils from "../test-utils";
+import sdk from "../..";
+import expect from 'expect';
+
+const SyncAccumulator = sdk.SyncAccumulator;
+
+describe("SyncAccumulator", function() {
+    let sa;
+
+    beforeEach(function() {
+        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        sa = new SyncAccumulator({
+            maxTimelineEntries: 10,
+        });
+    });
+
+    it("should return the same /sync response if accumulated exactly once", () => {
+        // technically cheating since we also cheekily pre-populate keys we
+        // know that the sync accumulator will pre-populate.
+        // It isn't 100% transitive.
+        const res = {
+            next_batch: "abc",
+            rooms: {
+                invite: {},
+                leave: {},
+                join: {
+                    "!foo:bar": {
+                        account_data: { events: [] },
+                        ephemeral: { events: [] },
+                        unread_notifications: {},
+                        state: {
+                            events: [
+                                member("alice", "join"),
+                                member("bob", "join"),
+                            ],
+                        },
+                        summary: {
+                            "m.heroes": undefined,
+                            "m.joined_member_count": undefined,
+                            "m.invited_member_count": undefined,
+                        },
+                        timeline: {
+                            events: [msg("alice", "hi")],
+                            prev_batch: "something",
+                        },
+                    },
+                },
+            },
+        };
+        sa.accumulate(res);
+        const output = sa.getJSON();
+        expect(output.nextBatch).toEqual(res.next_batch);
+        expect(output.roomsData).toEqual(res.rooms);
+    });
+
+    it("should prune the timeline to the oldest prev_batch within the limit", () => {
+        // maxTimelineEntries is 10 so we should get back all
+        // 10 timeline messages with a prev_batch of "pinned_to_1"
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                    msg("alice", "4"),
+                    msg("alice", "5"),
+                    msg("alice", "6"),
+                    msg("alice", "7"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "8"),
+                ],
+                prev_batch: "pinned_to_8",
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "9"),
+                    msg("alice", "10"),
+                ],
+                prev_batch: "pinned_to_10",
+            },
+        }));
+
+        let output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+1));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_1");
+
+        // accumulate more messages. Now it can't have a prev_batch of "pinned to 1"
+        // AND give us <= 10 messages without losing messages in-between.
+        // It should try to find the oldest prev_batch which still fits into 10
+        // messages, which is "pinned to 8".
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                events: [
+                    msg("alice", "11"),
+                    msg("alice", "12"),
+                    msg("alice", "13"),
+                    msg("alice", "14"),
+                    msg("alice", "15"),
+                    msg("alice", "16"),
+                    msg("alice", "17"),
+                ],
+                prev_batch: "pinned_to_11",
+            },
+        }));
+
+        output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(10);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+8));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_8");
+    });
+
+    it("should remove the stored timeline on limited syncs", () => {
+        sa.accumulate(syncSkeleton({
+            state: { events: [member("alice", "join")] },
+            timeline: {
+                events: [
+                    msg("alice", "1"),
+                    msg("alice", "2"),
+                    msg("alice", "3"),
+                ],
+                prev_batch: "pinned_to_1",
+            },
+        }));
+        // some time passes and now we get a limited sync
+        sa.accumulate(syncSkeleton({
+            state: { events: [] },
+            timeline: {
+                limited: true,
+                events: [
+                    msg("alice", "51"),
+                    msg("alice", "52"),
+                    msg("alice", "53"),
+                ],
+                prev_batch: "pinned_to_51",
+            },
+        }));
+
+        const output = sa.getJSON().roomsData.join["!foo:bar"];
+
+        expect(output.timeline.events.length).toEqual(3);
+        output.timeline.events.forEach((e, i) => {
+            expect(e.content.body).toEqual(""+(i+51));
+        });
+        expect(output.timeline.prev_batch).toEqual("pinned_to_51");
+    });
+
+    it("should drop typing notifications", () => {
+        const res = syncSkeleton({
+            ephemeral: {
+                events: [{
+                    type: "m.typing",
+                    content: {
+                        user_ids: ["@alice:localhost"],
+                    },
+                    room_id: "!foo:bar",
+                }],
+            },
+        });
+        sa.accumulate(res);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(0);
+    });
+
+    it("should clobber account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            account_data: {
+                events: [acc2],
+            },
+        }));
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].account_data.events[0],
+        ).toEqual(acc2);
+    });
+
+    it("should clobber global account data based on event type", () => {
+        const acc1 = {
+            type: "favourite.food",
+            content: {
+                food: "banana",
+            },
+        };
+        const acc2 = {
+            type: "favourite.food",
+            content: {
+                food: "apple",
+            },
+        };
+        sa.accumulate({
+            account_data: {
+                events: [acc1],
+            },
+        });
+        sa.accumulate({
+            account_data: {
+                events: [acc2],
+            },
+        });
+        expect(
+            sa.getJSON().accountData.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().accountData[0],
+        ).toEqual(acc2);
+    });
+
+    it("should accumulate read receipts", () => {
+        const receipt1 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                        "@bob:localhost": { ts: 2 },
+                    },
+                    "some.other.receipt.type": {
+                        "@should_be_ignored:localhost": { key: "val" },
+                    },
+                },
+            },
+        };
+        const receipt2 = {
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 }, // clobbers event1 receipt
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        };
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt1],
+            },
+        }));
+        sa.accumulate(syncSkeleton({
+            ephemeral: {
+                events: [receipt2],
+            },
+        }));
+
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events.length,
+        ).toEqual(1);
+        expect(
+            sa.getJSON().roomsData.join["!foo:bar"].ephemeral.events[0],
+        ).toEqual({
+            type: "m.receipt",
+            room_id: "!foo:bar",
+            content: {
+                "$event1:localhost": {
+                    "m.read": {
+                        "@alice:localhost": { ts: 1 },
+                    },
+                },
+                "$event2:localhost": {
+                    "m.read": {
+                        "@bob:localhost": { ts: 2 },
+                        "@charlie:localhost": { ts: 3 },
+                    },
+                },
+            },
+        });
+    });
+
+    describe("summary field", function() {
+        function createSyncResponseWithSummary(summary) {
+            return {
+                next_batch: "abc",
+                rooms: {
+                    invite: {},
+                    leave: {},
+                    join: {
+                        "!foo:bar": {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            unread_notifications: {},
+                            state: {
+                                events: [],
+                            },
+                            summary: summary,
+                            timeline: {
+                                events: [],
+                                prev_batch: "something",
+                            },
+                        },
+                    },
+                },
+            };
+        }
+
+        it("should copy summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.heroes"]).toEqual(["@alice:bar"]);
+        });
+
+        it("should accumulate summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@bob:bar"],
+                "m.joined_member_count": 5,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.joined_member_count"]).toEqual(5);
+            expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
+        });
+    });
+});
+
+function syncSkeleton(joinObj) {
+    joinObj = joinObj || {};
+    return {
+        next_batch: "abc",
+        rooms: {
+            join: {
+                "!foo:bar": joinObj,
+            },
+        },
+    };
+}
+
+function msg(localpart, text) {
+    return {
+        content: {
+            body: text,
+        },
+        origin_server_ts: 123456789,
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.message",
+    };
+}
+
+function member(localpart, membership) {
+    return {
+        content: {
+            membership: membership,
+        },
+        origin_server_ts: 123456789,
+        state_key: "@" + localpart + ":localhost",
+        sender: "@" + localpart + ":localhost",
+        type: "m.room.member",
+    };
+}

spec/unit/timeline-window.spec.js

@@ -1,28 +1,36 @@
 "use strict";
-var q = require("q");
-var sdk = require("../..");
-var EventTimeline = sdk.EventTimeline;
-var TimelineWindow = sdk.TimelineWindow;
-var TimelineIndex = require("../../lib/timeline-window").TimelineIndex;
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const EventTimeline = sdk.EventTimeline;
+const TimelineWindow = sdk.TimelineWindow;
+const TimelineIndex = require("../../lib/timeline-window").TimelineIndex;
 
-var utils = require("../test-utils");
+const utils = require("../test-utils");
+import expect from 'expect';
 
-var ROOM_ID = "roomId";
-var USER_ID = "userId";
+const ROOM_ID = "roomId";
+const USER_ID = "userId";
 
 /*
  * create a timeline with a bunch (default 3) events.
  * baseIndex is 1 by default.
  */
 function createTimeline(numEvents, baseIndex) {
-    if (numEvents === undefined) { numEvents = 3; }
-    if (baseIndex === undefined) { baseIndex = 1; }
+    if (numEvents === undefined) {
+        numEvents = 3;
+    }
+    if (baseIndex === undefined) {
+        baseIndex = 1;
+    }
 
     // XXX: this is a horrid hack
-    var timelineSet = { room: { roomId: ROOM_ID }};
-    timelineSet.room.getUnfilteredTimelineSet = function() { return timelineSet; };
+    const timelineSet = { room: { roomId: ROOM_ID }};
+    timelineSet.room.getUnfilteredTimelineSet = function() {
+        return timelineSet;
+    };
 
-    var timeline = new EventTimeline(timelineSet);
+    const timeline = new EventTimeline(timelineSet);
 
     // add the events after the baseIndex first
     addEventsToTimeline(timeline, numEvents - baseIndex, false);
@@ -35,12 +43,12 @@ function createTimeline(numEvents, baseIndex) {
 }
 
 function addEventsToTimeline(timeline, numEvents, atStart) {
-    for (var i = 0; i < numEvents; i++) {
+    for (let i = 0; i < numEvents; i++) {
         timeline.addEvent(
             utils.mkMessage({
                 room: ROOM_ID, user: USER_ID,
                 event: true,
-            }), atStart
+            }), atStart,
         );
     }
 }
@@ -50,8 +58,8 @@ function addEventsToTimeline(timeline, numEvents, atStart) {
  * create a pair of linked timelines
  */
 function createLinkedTimelines() {
-    var tl1 = createTimeline();
-    var tl2 = createTimeline();
+    const tl1 = createTimeline();
+    const tl2 = createTimeline();
     tl1.setNeighbouringTimeline(tl2, EventTimeline.FORWARDS);
     tl2.setNeighbouringTimeline(tl1, EventTimeline.BACKWARDS);
     return [tl1, tl2];
@@ -65,41 +73,42 @@ describe("TimelineIndex", function() {
 
     describe("minIndex", function() {
         it("should return the min index relative to BaseIndex", function() {
-            var timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
             expect(timelineIndex.minIndex()).toEqual(-1);
         });
     });
 
     describe("maxIndex", function() {
         it("should return the max index relative to BaseIndex", function() {
-            var timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
             expect(timelineIndex.maxIndex()).toEqual(2);
         });
     });
 
     describe("advance", function() {
         it("should advance up to the end of the timeline", function() {
-            var timelineIndex = new TimelineIndex(createTimeline(), 0);
-            var result = timelineIndex.advance(3);
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.advance(3);
             expect(result).toEqual(2);
             expect(timelineIndex.index).toEqual(2);
         });
 
         it("should retreat back to the start of the timeline", function() {
-            var timelineIndex = new TimelineIndex(createTimeline(), 0);
-            var result = timelineIndex.advance(-2);
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.advance(-2);
             expect(result).toEqual(-1);
             expect(timelineIndex.index).toEqual(-1);
         });
 
         it("should advance into the next timeline", function() {
-            var timelines = createLinkedTimelines();
-            var tl1 = timelines[0], tl2 = timelines[1];
+            const timelines = createLinkedTimelines();
+            const tl1 = timelines[0];
+            const tl2 = timelines[1];
 
             // initialise the index pointing at the end of the first timeline
-            var timelineIndex = new TimelineIndex(tl1, 2);
+            const timelineIndex = new TimelineIndex(tl1, 2);
 
-            var result = timelineIndex.advance(1);
+            const result = timelineIndex.advance(1);
             expect(result).toEqual(1);
             expect(timelineIndex.timeline).toBe(tl2);
 
@@ -110,14 +119,15 @@ describe("TimelineIndex", function() {
         });
 
         it("should retreat into the previous timeline", function() {
-            var timelines = createLinkedTimelines();
-            var tl1 = timelines[0], tl2 = timelines[1];
+            const timelines = createLinkedTimelines();
+            const tl1 = timelines[0];
+            const tl2 = timelines[1];
 
             // initialise the index pointing at the start of the second
             // timeline
-            var timelineIndex = new TimelineIndex(tl2, -1);
+            const timelineIndex = new TimelineIndex(tl2, -1);
 
-            var result = timelineIndex.advance(-1);
+            const result = timelineIndex.advance(-1);
             expect(result).toEqual(-1);
             expect(timelineIndex.timeline).toBe(tl1);
             expect(timelineIndex.index).toEqual(1);
@@ -126,8 +136,8 @@ describe("TimelineIndex", function() {
 
     describe("retreat", function() {
         it("should retreat up to the start of the timeline", function() {
-            var timelineIndex = new TimelineIndex(createTimeline(), 0);
-            var result = timelineIndex.retreat(2);
+            const timelineIndex = new TimelineIndex(createTimeline(), 0);
+            const result = timelineIndex.retreat(2);
             expect(result).toEqual(1);
             expect(timelineIndex.index).toEqual(-1);
         });
@@ -140,13 +150,14 @@ describe("TimelineWindow", function() {
      * create a dummy eventTimelineSet and client, and a TimelineWindow
      * attached to them.
      */
-    var timelineSet, client;
+    let timelineSet;
+    let client;
     function createWindow(timeline, opts) {
         timelineSet = {};
         client = {};
         client.getEventTimeline = function(timelineSet0, eventId0) {
             expect(timelineSet0).toBe(timelineSet);
-            return q(timeline);
+            return Promise.resolve(timeline);
         };
 
         return new TimelineWindow(client, timelineSet, opts);
@@ -158,77 +169,79 @@ describe("TimelineWindow", function() {
 
     describe("load", function() {
         it("should initialise from the live timeline", function(done) {
-            var liveTimeline = createTimeline();
-            var room = {};
-            room.getLiveTimeline = function() { return liveTimeline; };
+            const liveTimeline = createTimeline();
+            const room = {};
+            room.getLiveTimeline = function() {
+                return liveTimeline;
+            };
 
-            var timelineWindow = new TimelineWindow(undefined, room);
+            const timelineWindow = new TimelineWindow(undefined, room);
             timelineWindow.load(undefined, 2).then(function() {
-                var expectedEvents = liveTimeline.getEvents().slice(1);
+                const expectedEvents = liveTimeline.getEvents().slice(1);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should initialise from a specific event", function(done) {
-            var timeline = createTimeline();
-            var eventId = timeline.getEvents()[1].getId();
+            const timeline = createTimeline();
+            const eventId = timeline.getEvents()[1].getId();
 
-            var timelineSet = {};
-            var client = {};
+            const timelineSet = {};
+            const client = {};
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineSet0).toBe(timelineSet);
                 expect(eventId0).toEqual(eventId);
-                return q(timeline);
+                return Promise.resolve(timeline);
             };
 
-            var timelineWindow = new TimelineWindow(client, timelineSet);
+            const timelineWindow = new TimelineWindow(client, timelineSet);
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("canPaginate should return false until load has returned",
            function(done) {
-            var timeline = createTimeline();
+            const timeline = createTimeline();
             timeline.setPaginationToken("toktok1", EventTimeline.BACKWARDS);
             timeline.setPaginationToken("toktok2", EventTimeline.FORWARDS);
 
-            var eventId = timeline.getEvents()[1].getId();
+            const eventId = timeline.getEvents()[1].getId();
 
-            var timelineSet = {};
-            var client = {};
+            const timelineSet = {};
+            const client = {};
 
-            var timelineWindow = new TimelineWindow(client, timelineSet);
+            const timelineWindow = new TimelineWindow(client, timelineSet);
 
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(false);
-                return q(timeline);
+                return Promise.resolve(timeline);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
                     .toBe(true);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
     });
 
     describe("pagination", function() {
         it("should be able to advance across the initial timeline",
            function(done) {
-            var timeline = createTimeline();
-            var eventId = timeline.getEvents()[1].getId();
-            var timelineWindow = createWindow(timeline);
+            const timeline = createTimeline();
+            const eventId = timeline.getEvents()[1].getId();
+            const timelineWindow = createWindow(timeline);
 
             timelineWindow.load(eventId, 1).then(function() {
-                var expectedEvents = [timeline.getEvents()[1]];
+                const expectedEvents = [timeline.getEvents()[1]];
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -239,7 +252,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = timeline.getEvents().slice(1);
+                const expectedEvents = timeline.getEvents().slice(1);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -254,7 +267,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -264,16 +277,16 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should advance into next timeline", function(done) {
-            var tls = createLinkedTimelines();
-            var eventId = tls[0].getEvents()[1].getId();
-            var timelineWindow = createWindow(tls[0], {windowLimit: 5});
+            const tls = createLinkedTimelines();
+            const eventId = tls[0].getEvents()[1].getId();
+            const timelineWindow = createWindow(tls[0], {windowLimit: 5});
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = tls[0].getEvents();
+                const expectedEvents = tls[0].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -284,7 +297,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = tls[0].getEvents()
+                const expectedEvents = tls[0].getEvents()
                     .concat(tls[1].getEvents().slice(0, 2));
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -298,7 +311,7 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 // the windowLimit should have made us drop an event from
                 // tls[0]
-                var expectedEvents = tls[0].getEvents().slice(1)
+                const expectedEvents = tls[0].getEvents().slice(1)
                     .concat(tls[1].getEvents());
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -309,16 +322,16 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should retreat into previous timeline", function(done) {
-            var tls = createLinkedTimelines();
-            var eventId = tls[1].getEvents()[1].getId();
-            var timelineWindow = createWindow(tls[1], {windowLimit: 5});
+            const tls = createLinkedTimelines();
+            const eventId = tls[1].getEvents()[1].getId();
+            const timelineWindow = createWindow(tls[1], {windowLimit: 5});
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = tls[1].getEvents();
+                const expectedEvents = tls[1].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -329,7 +342,7 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = tls[0].getEvents().slice(1, 3)
+                const expectedEvents = tls[0].getEvents().slice(1, 3)
                     .concat(tls[1].getEvents());
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -343,7 +356,7 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 // the windowLimit should have made us drop an event from
                 // tls[1]
-                var expectedEvents = tls[0].getEvents()
+                const expectedEvents = tls[0].getEvents()
                     .concat(tls[1].getEvents().slice(0, 2));
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -354,15 +367,15 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should make forward pagination requests", function(done) {
-            var timeline = createTimeline();
+            const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
-            var timelineWindow = createWindow(timeline, {windowLimit: 5});
-            var eventId = timeline.getEvents()[1].getId();
+            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const eventId = timeline.getEvents()[1].getId();
 
             client.paginateEventTimeline = function(timeline0, opts) {
                 expect(timeline0).toBe(timeline);
@@ -370,11 +383,11 @@ describe("TimelineWindow", function() {
                 expect(opts.limit).toEqual(2);
 
                 addEventsToTimeline(timeline, 3, false);
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -384,18 +397,18 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = timeline.getEvents().slice(0, 5);
+                const expectedEvents = timeline.getEvents().slice(0, 5);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
 
         it("should make backward pagination requests", function(done) {
-            var timeline = createTimeline();
+            const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
 
-            var timelineWindow = createWindow(timeline, {windowLimit: 5});
-            var eventId = timeline.getEvents()[1].getId();
+            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const eventId = timeline.getEvents()[1].getId();
 
             client.paginateEventTimeline = function(timeline0, opts) {
                 expect(timeline0).toBe(timeline);
@@ -403,11 +416,11 @@ describe("TimelineWindow", function() {
                 expect(opts.limit).toEqual(2);
 
                 addEventsToTimeline(timeline, 3, true);
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -417,30 +430,30 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(true);
-                var expectedEvents = timeline.getEvents().slice(1, 6);
+                const expectedEvents = timeline.getEvents().slice(1, 6);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
 
         it("should limit the number of unsuccessful pagination requests",
         function(done) {
-            var timeline = createTimeline();
+            const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
-            var timelineWindow = createWindow(timeline, {windowLimit: 5});
-            var eventId = timeline.getEvents()[1].getId();
+            const timelineWindow = createWindow(timeline, {windowLimit: 5});
+            const eventId = timeline.getEvents()[1].getId();
 
-            var paginateCount = 0;
+            let paginateCount = 0;
             client.paginateEventTimeline = function(timeline0, opts) {
                 expect(timeline0).toBe(timeline);
                 expect(opts.backwards).toBe(false);
                 expect(opts.limit).toEqual(2);
                 paginateCount += 1;
-                return q(true);
+                return Promise.resolve(true);
             };
 
             timelineWindow.load(eventId, 3).then(function() {
-                var expectedEvents = timeline.getEvents();
+                const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
@@ -451,14 +464,14 @@ describe("TimelineWindow", function() {
             }).then(function(success) {
                 expect(success).toBe(false);
                 expect(paginateCount).toEqual(3);
-                var expectedEvents = timeline.getEvents().slice(0, 3);
+                const expectedEvents = timeline.getEvents().slice(0, 3);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
         });
     });
 });

spec/unit/user.spec.js

@@ -1,11 +1,14 @@
 "use strict";
-var sdk = require("../..");
-var User = sdk.User;
-var utils = require("../test-utils");
+import 'source-map-support/register';
+const sdk = require("../..");
+const User = sdk.User;
+const utils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("User", function() {
-    var userId = "@alice:bar";
-    var user;
+    const userId = "@alice:bar";
+    let user;
 
     beforeEach(function() {
         utils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -13,18 +16,18 @@ describe("User", function() {
     });
 
     describe("setPresenceEvent", function() {
-        var event = utils.mkEvent({
+        const event = utils.mkEvent({
             type: "m.presence", content: {
                 presence: "online",
                 user_id: userId,
                 displayname: "Alice",
                 last_active_ago: 1085,
-                avatar_url: "mxc://foo/bar"
-            }, event: true
+                avatar_url: "mxc://foo/bar",
+            }, event: true,
         });
 
         it("should emit 'User.displayName' if the display name changes", function() {
-            var emitCount = 0;
+            let emitCount = 0;
             user.on("User.displayName", function(ev, usr) {
                 emitCount += 1;
             });
@@ -35,7 +38,7 @@ describe("User", function() {
         });
 
         it("should emit 'User.avatarUrl' if the avatar URL changes", function() {
-            var emitCount = 0;
+            let emitCount = 0;
             user.on("User.avatarUrl", function(ev, usr) {
                 emitCount += 1;
             });
@@ -46,7 +49,7 @@ describe("User", function() {
         });
 
         it("should emit 'User.presence' if the presence changes", function() {
-            var emitCount = 0;
+            let emitCount = 0;
             user.on("User.presence", function(ev, usr) {
                 emitCount += 1;
             });

spec/unit/utils.spec.js

@@ -1,6 +1,9 @@
 "use strict";
-var utils = require("../../lib/utils");
-var testUtils = require("../test-utils");
+import 'source-map-support/register';
+const utils = require("../../lib/utils");
+const testUtils = require("../test-utils");
+
+import expect from 'expect';
 
 describe("utils", function() {
     beforeEach(function() {
@@ -9,32 +12,32 @@ describe("utils", function() {
 
     describe("encodeParams", function() {
         it("should url encode and concat with &s", function() {
-            var params = {
+            const params = {
                 foo: "bar",
-                baz: "beer@"
+                baz: "beer@",
             };
             expect(utils.encodeParams(params)).toEqual(
-                "foo=bar&baz=beer%40"
+                "foo=bar&baz=beer%40",
             );
         });
     });
 
     describe("encodeUri", function() {
         it("should replace based on object keys and url encode", function() {
-            var path = "foo/bar/%something/%here";
-            var vals = {
+            const path = "foo/bar/%something/%here";
+            const vals = {
                 "%something": "baz",
-                "%here": "beer@"
+                "%here": "beer@",
             };
             expect(utils.encodeUri(path, vals)).toEqual(
-                "foo/bar/baz/beer%40"
+                "foo/bar/baz/beer%40",
             );
         });
     });
 
     describe("forEach", function() {
         it("should be invoked for each element", function() {
-            var arr = [];
+            const arr = [];
             utils.forEach([55, 66, 77], function(element) {
                 arr.push(element);
             });
@@ -44,38 +47,50 @@ describe("utils", function() {
 
     describe("findElement", function() {
         it("should find only 1 element if there is a match", function() {
-            var matchFn = function() { return true; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
             expect(utils.findElement(arr, matchFn)).toEqual(55);
         });
         it("should be able to find in reverse order", function() {
-            var matchFn = function() { return true; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
             expect(utils.findElement(arr, matchFn, true)).toEqual(77);
         });
         it("should find nothing if the function never returns true", function() {
-            var matchFn = function() { return false; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return false;
+            };
+            const arr = [55, 66, 77];
             expect(utils.findElement(arr, matchFn)).toBeFalsy();
         });
     });
 
     describe("removeElement", function() {
         it("should remove only 1 element if there is a match", function() {
-            var matchFn = function() { return true; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
             utils.removeElement(arr, matchFn);
             expect(arr).toEqual([66, 77]);
         });
         it("should be able to remove in reverse order", function() {
-            var matchFn = function() { return true; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return true;
+            };
+            const arr = [55, 66, 77];
             utils.removeElement(arr, matchFn, true);
             expect(arr).toEqual([55, 66]);
         });
         it("should remove nothing if the function never returns true", function() {
-            var matchFn = function() { return false; };
-            var arr = [55, 66, 77];
+            const matchFn = function() {
+                return false;
+            };
+            const arr = [55, 66, 77];
             utils.removeElement(arr, matchFn);
             expect(arr).toEqual(arr);
         });
@@ -92,7 +107,7 @@ describe("utils", function() {
             expect(utils.isFunction(555)).toBe(false);
 
             expect(utils.isFunction(function() {})).toBe(true);
-            var s = { foo: function() {} };
+            const s = { foo: function() {} };
             expect(utils.isFunction(s.foo)).toBe(true);
         });
     });
@@ -113,30 +128,42 @@ describe("utils", function() {
 
     describe("checkObjectHasKeys", function() {
         it("should throw for missing keys", function() {
-            expect(function() { utils.checkObjectHasKeys({}, ["foo"]); }).toThrow();
-            expect(function() { utils.checkObjectHasKeys({
-                foo: "bar"
-            }, ["foo"]); }).not.toThrow();
+            expect(function() {
+                utils.checkObjectHasKeys({}, ["foo"]);
+            }).toThrow();
+            expect(function() {
+                utils.checkObjectHasKeys({
+                    foo: "bar",
+                }, ["foo"]);
+            }).toNotThrow();
         });
     });
 
     describe("checkObjectHasNoAdditionalKeys", function() {
         it("should throw for extra keys", function() {
-            expect(function() { utils.checkObjectHasNoAdditionalKeys({
-                foo: "bar",
-                baz: 4
-            }, ["foo"]); }).toThrow();
+            expect(function() {
+                utils.checkObjectHasNoAdditionalKeys({
+                            foo: "bar",
+                            baz: 4,
+                        }, ["foo"]);
+            }).toThrow();
 
-            expect(function() { utils.checkObjectHasNoAdditionalKeys({
-                foo: "bar"
-            }, ["foo"]); }).not.toThrow();
+            expect(function() {
+                utils.checkObjectHasNoAdditionalKeys({
+                            foo: "bar",
+                        }, ["foo"]);
+            }).toNotThrow();
         });
     });
 
     describe("deepCompare", function() {
-        var assert = {
-            isTrue: function(x) { expect(x).toBe(true); },
-            isFalse: function(x) { expect(x).toBe(false); },
+        const assert = {
+            isTrue: function(x) {
+                expect(x).toBe(true);
+            },
+            isFalse: function(x) {
+                expect(x).toBe(false);
+            },
         };
 
         it("should handle primitives", function() {
@@ -150,7 +177,7 @@ describe("utils", function() {
         it("should handle regexps", function() {
             assert.isTrue(utils.deepCompare(/abc/, /abc/));
             assert.isFalse(utils.deepCompare(/abc/, /123/));
-            var r = /abc/;
+            const r = /abc/;
             assert.isTrue(utils.deepCompare(r, r));
         });
 
@@ -192,8 +219,12 @@ describe("utils", function() {
             // no two different function is equal really, they capture their
             // context variables so even if they have same toString(), they
             // won't have same functionality
-            var func = function(x) { return true; };
-            var func2 = function(x) { return true; };
+            const func = function(x) {
+                return true;
+            };
+            const func2 = function(x) {
+                return true;
+            };
             assert.isTrue(utils.deepCompare(func, func));
             assert.isFalse(utils.deepCompare(func, func2));
             assert.isTrue(utils.deepCompare({ a: { b: func } }, { a: { b: func } }));
@@ -203,58 +234,58 @@ describe("utils", function() {
 
 
     describe("extend", function() {
-        var SOURCE = { "prop2": 1, "string2": "x", "newprop": "new" };
+        const SOURCE = { "prop2": 1, "string2": "x", "newprop": "new" };
 
         it("should extend", function() {
-            var target = {
+            const target = {
                 "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
             };
-            var merged = {
+            const merged = {
                 "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
                 "newprop": "new",
             };
-            var source_orig = JSON.stringify(SOURCE);
+            const sourceOrig = JSON.stringify(SOURCE);
 
             utils.extend(target, SOURCE);
             expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
 
             // check the originial wasn't modified
-            expect(JSON.stringify(SOURCE)).toEqual(source_orig);
+            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
         });
 
         it("should ignore null", function() {
-            var target = {
+            const target = {
                 "prop1": 5, "prop2": 7, "string1": "baz", "string2": "foo",
             };
-            var merged = {
+            const merged = {
                 "prop1": 5, "prop2": 1, "string1": "baz", "string2": "x",
                 "newprop": "new",
             };
-            var source_orig = JSON.stringify(SOURCE);
+            const sourceOrig = JSON.stringify(SOURCE);
 
             utils.extend(target, null, SOURCE);
             expect(JSON.stringify(target)).toEqual(JSON.stringify(merged));
 
             // check the originial wasn't modified
-            expect(JSON.stringify(SOURCE)).toEqual(source_orig);
+            expect(JSON.stringify(SOURCE)).toEqual(sourceOrig);
         });
 
         it("should handle properties created with defineProperties", function() {
-            var source = Object.defineProperties({}, {
+            const source = Object.defineProperties({}, {
                 "enumerableProp": {
                     get: function() {
                         return true;
                     },
-                    enumerable: true
+                    enumerable: true,
                 },
                 "nonenumerableProp": {
                     get: function() {
                         return true;
-                    }
-                }
+                    },
+                },
             });
 
-            var target = {};
+            const target = {};
             utils.extend(target, source);
             expect(target.enumerableProp).toBe(true);
             expect(target.nonenumerableProp).toBe(undefined);

spec/unit/webstorage.spec.js

@@ -1,580 +0,0 @@
-"use strict";
-var sdk = require("../..");
-var WebStorageStore = sdk.WebStorageStore;
-var Room = sdk.Room;
-var User = sdk.User;
-var utils = require("../test-utils");
-
-var MockStorageApi = require("../MockStorageApi");
-
-describe("WebStorageStore", function() {
-    var store, room;
-    var roomId = "!foo:bar";
-    var userId = "@alice:bar";
-    var mockStorageApi;
-    var batchNum = 3;
-    // web storage api keys
-    var prefix = "room_" + roomId + "_timeline_";
-    var stateKeyName = "room_" + roomId + "_state";
-
-    // stored state events
-    var stateEventMap = {
-        "m.room.member": {},
-        "m.room.name": {}
-    };
-    stateEventMap["m.room.member"][userId] = utils.mkMembership(
-        {user: userId, room: roomId, mship: "join"}
-    );
-    stateEventMap["m.room.name"][""] = utils.mkEvent(
-        {user: userId, room: roomId, type: "m.room.name",
-        content: {
-            name: "foo"
-        }}
-    );
-
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line no-invalid-this
-        mockStorageApi = new MockStorageApi();
-        store = new WebStorageStore(mockStorageApi, batchNum);
-        room = new Room(roomId);
-    });
-
-    describe("constructor", function() {
-        it("should throw if the WebStorage API functions are missing", function() {
-            expect(function() {
-                store = new WebStorageStore({}, 5);
-            }).toThrow();
-            expect(function() {
-                mockStorageApi.length = undefined;
-                store = new WebStorageStore(mockStorageApi, 5);
-            }).toThrow();
-        });
-    });
-
-    describe("syncToken", function() {
-        it("get: should return the token from the store", function() {
-            var token = "flibble";
-            store.setSyncToken(token);
-            expect(store.getSyncToken()).toEqual(token);
-            expect(mockStorageApi.length).toEqual(1);
-        });
-        it("get: should return null if the token does not exist", function() {
-            expect(store.getSyncToken()).toEqual(null);
-            expect(mockStorageApi.length).toEqual(0);
-        });
-    });
-
-    describe("storeRoom", function() {
-        it("should persist the room state correctly", function() {
-            var stateEvents = [
-                utils.mkEvent({
-                    event: true, type: "m.room.create", user: userId, room: roomId,
-                    content: {
-                        creator: userId
-                    }
-                }),
-                utils.mkMembership({
-                    event: true, user: userId, room: roomId, mship: "join"
-                })
-            ];
-            room.currentState.setStateEvents(stateEvents);
-            store.storeRoom(room);
-            var storedEvents = getItem(mockStorageApi,
-                "room_" + roomId + "_state"
-            ).events;
-            expect(storedEvents["m.room.create"][""]).toEqual(stateEvents[0].event);
-        });
-
-        it("should persist timeline events correctly", function() {
-            var timelineEvents = [];
-            var entries = batchNum + batchNum - 1;
-            var i = 0;
-            for (i = 0; i < entries; i++) {
-                timelineEvents.push(
-                    utils.mkMessage({room: roomId, user: userId, event: true})
-                );
-            }
-            room.timeline = timelineEvents;
-            store.storeRoom(room);
-            expect(getItem(mockStorageApi, prefix + "-1")).toBe(null);
-            expect(getItem(mockStorageApi, prefix + "2")).toBe(null);
-            expect(getItem(mockStorageApi, prefix + "live")).toBe(null);
-            var timeline0 = getItem(mockStorageApi, prefix + "0");
-            var timeline1 = getItem(mockStorageApi, prefix + "1");
-            expect(timeline0.length).toEqual(batchNum);
-            expect(timeline1.length).toEqual(batchNum - 1);
-            for (i = 0; i < batchNum; i++) {
-                expect(timeline0[i]).toEqual(timelineEvents[i].event);
-                if ((i + batchNum) < timelineEvents.length) {
-                    expect(timeline1[i]).toEqual(timelineEvents[i + batchNum].event);
-                }
-            }
-        });
-
-        it("should persist timeline events in one bucket if batchNum=0", function() {
-            store = new WebStorageStore(mockStorageApi, 0);
-            var timelineEvents = [];
-            var entries = batchNum + batchNum - 1;
-            var i = 0;
-            for (i = 0; i < entries; i++) {
-                timelineEvents.push(
-                    utils.mkMessage({room: roomId, user: userId, event: true})
-                );
-            }
-            room.timeline = timelineEvents;
-            store.storeRoom(room);
-            expect(getItem(mockStorageApi, prefix + "-1")).toBe(null);
-            expect(getItem(mockStorageApi, prefix + "1")).toBe(null);
-            expect(getItem(mockStorageApi, prefix + "live")).toBe(null);
-            var timeline = getItem(mockStorageApi, prefix + "0");
-            expect(timeline.length).toEqual(timelineEvents.length);
-            for (i = 0; i < timeline.length; i++) {
-                expect(timeline[i]).toEqual(
-                    timelineEvents[i].event
-                );
-            }
-        });
-    });
-
-    describe("getRoom", function() {
-        // stored timeline events
-        var timeline0, timeline1, i;
-
-        beforeEach(function() {
-            timeline0 = [];
-            timeline1 = [];
-            for (i = 0; i < batchNum; i++) {
-                timeline1[i] = utils.mkMessage({user: userId, room: roomId});
-                if (i !== (batchNum - 1)) { // miss last one
-                    timeline0[i] = utils.mkMessage({user: userId, room: roomId});
-                }
-            }
-        });
-
-        it("should reconstruct room state", function() {
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-
-            var storedRoom = store.getRoom(roomId);
-            expect(
-                storedRoom.currentState.getStateEvents("m.room.name", "").event
-            ).toEqual(stateEventMap["m.room.name"][""]);
-            expect(
-                storedRoom.currentState.getStateEvents("m.room.member", userId).event
-            ).toEqual(stateEventMap["m.room.member"][userId]);
-        });
-
-        it("should reconstruct old room state", function() {
-            var inviteEvent = utils.mkMembership({
-                user: userId, room: roomId, mship: "invite"
-            });
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", [inviteEvent]);
-
-            var storedRoom = store.getRoom(roomId);
-            expect(
-                storedRoom.currentState.getStateEvents("m.room.member", userId).event
-            ).toEqual(stateEventMap["m.room.member"][userId]);
-            expect(
-                storedRoom.oldState.getStateEvents("m.room.member", userId).event
-            ).toEqual(inviteEvent);
-        });
-
-        it("should reconstruct the room timeline", function() {
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", timeline0);
-            setItem(mockStorageApi, prefix + "1", timeline1);
-
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom).not.toBeNull();
-            // should only get up to the batch num timeline events
-            expect(storedRoom.timeline.length).toEqual(batchNum);
-            var timeline = timeline0.concat(timeline1);
-            for (i = 0; i < batchNum; i++) {
-                expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual(
-                    timeline[timeline.length - 1 - i]
-                );
-            }
-        });
-
-        it("should sync the timeline for 'live' events " +
-        "(full hi batch; 1+bit live batches)", function() {
-            // 1 and a bit events go into _live
-            var timelineLive = [];
-            timelineLive.push(utils.mkMessage({user: userId, room: roomId}));
-            for (i = 0; i < batchNum; i++) {
-                timelineLive.push(
-                    utils.mkMessage({user: userId, room: roomId})
-                );
-            }
-
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", timeline0);
-            setItem(mockStorageApi, prefix + "1", timeline1);
-            setItem(mockStorageApi,
-                // deep copy the timeline via parse/stringify else items will
-                // be shift()ed from timelineLive and we can't compare!
-                prefix + "live", JSON.parse(JSON.stringify(timelineLive))
-            );
-
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom).not.toBeNull();
-            // should only get up to the batch num timeline events (highest
-            // index of timelineLive is the newest message)
-            expect(storedRoom.timeline.length).toEqual(batchNum);
-            for (i = 0; i < batchNum; i++) {
-                expect(storedRoom.timeline[i].event).toEqual(
-                    timelineLive[i + 1]
-                );
-            }
-        });
-
-        it("should sync the timeline for 'live' events " +
-        "(no low batch; 1 live batches)", function() {
-            var timelineLive = [];
-            for (i = 0; i < batchNum; i++) {
-                timelineLive.push(
-                    utils.mkMessage({user: userId, room: roomId})
-                );
-            }
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", []);
-            setItem(mockStorageApi,
-                // deep copy the timeline via parse/stringify else items will
-                // be shift()ed from timelineLive and we can't compare!
-                prefix + "live", JSON.parse(JSON.stringify(timelineLive))
-            );
-
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom).not.toBeNull();
-            // should only get up to the batch num timeline events (highest
-            // index of timelineLive is the newest message)
-            expect(storedRoom.timeline.length).toEqual(batchNum);
-            for (i = 0; i < batchNum; i++) {
-                expect(storedRoom.timeline[i].event).toEqual(
-                    timelineLive[i]
-                );
-            }
-        });
-
-        it("should be able to reconstruct the timeline with negative indices",
-        function() {
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "-5", timeline0);
-            setItem(mockStorageApi, prefix + "-4", timeline1);
-            var timeline = timeline0.concat(timeline1);
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom).not.toBeNull();
-            // should only get up to the batch num timeline events
-            expect(storedRoom.timeline.length).toEqual(batchNum);
-            for (i = 0; i < batchNum; i++) {
-                expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual(
-                    timeline[timeline.length - 1 - i]
-                );
-            }
-        });
-
-        it("should return null if the room doesn't exist", function() {
-            expect(store.getRoom("nothing")).toEqual(null);
-        });
-
-        it("should assign a storageToken to the Room", function() {
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", timeline0);
-            setItem(mockStorageApi, prefix + "1", timeline1);
-
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom.storageToken).toBeDefined();
-        });
-    });
-
-    describe("scrollback", function() {
-        // stored timeline events
-        var timeline0, timeline1, timeline2;
-
-        beforeEach(function() {
-            // batch size is 3
-            store = new WebStorageStore(mockStorageApi, 3);
-            timeline0 = [
-                // _
-                utils.mkMessage({user: userId, room: roomId}), // 1  OLDEST
-                utils.mkMessage({user: userId, room: roomId})  // 2
-            ];
-            timeline1 = [
-                utils.mkMessage({user: userId, room: roomId}), // 3
-                utils.mkMessage({user: userId, room: roomId}), // 4
-                utils.mkMessage({user: userId, room: roomId})  // 5
-            ];
-            timeline2 = [
-                utils.mkMessage({user: userId, room: roomId}), // 6
-                utils.mkMessage({user: userId, room: roomId}), // 7
-                utils.mkMessage({user: userId, room: roomId})  // 8  NEWEST
-            ];
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", timeline0);
-            setItem(mockStorageApi, prefix + "1", timeline1);
-            setItem(mockStorageApi, prefix + "2", timeline2);
-        });
-
-        it("should scroll back locally giving 'limit' events", function() {
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom.timeline.length).toEqual(3);
-            var events = store.scrollback(storedRoom, 3);
-            expect(events.length).toEqual(3);
-            expect(events.reverse()).toEqual(timeline1);
-        });
-
-        it("should give less than 'limit' events near the end of the stored timeline",
-        function() {
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom.timeline.length).toEqual(3);
-            var events = store.scrollback(storedRoom, 7);
-            expect(events.length).toEqual(5);
-            expect(events.reverse()).toEqual(timeline0.concat(timeline1));
-        });
-
-        it("should progressively give older messages the more times scrollback is called",
-        function() {
-            var events;
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom.timeline.length).toEqual(3);
-
-            events = store.scrollback(storedRoom, 2);
-            expect(events.reverse()).toEqual([timeline1[1], timeline1[2]]);
-            expect(storedRoom.timeline.length).toEqual(5);
-
-            events = store.scrollback(storedRoom, 2);
-            expect(events.reverse()).toEqual([timeline0[1], timeline1[0]]);
-            expect(storedRoom.timeline.length).toEqual(7);
-
-            events = store.scrollback(storedRoom, 2);
-            expect(events).toEqual([timeline0[0]]);
-            expect(storedRoom.timeline.length).toEqual(8);
-
-            events = store.scrollback(storedRoom, 2);
-            expect(events).toEqual([]);
-            expect(storedRoom.timeline.length).toEqual(8);
-        });
-
-        it("should give 0 events if there is no token on the room", function() {
-            var r = new Room(roomId);
-            expect(store.scrollback(r, 3)).toEqual([]);
-        });
-
-        it("should give 0 events for unknown rooms", function() {
-            var r = new Room("!unknown:room");
-            r.storageToken = "foo";
-            expect(store.scrollback(r, 3)).toEqual([]);
-        });
-
-        it("should give 0 events if the boundary event is the last in the timeline",
-        function() {
-            var events;
-            var storedRoom = store.getRoom(roomId);
-            expect(storedRoom.timeline.length).toEqual(3);
-
-            // go up to the boundary (8 messages total)
-            events = store.scrollback(storedRoom, 5);
-            expect(events.length).toEqual(5);
-
-            events = store.scrollback(storedRoom, 5);
-            expect(events.length).toEqual(0);
-        });
-    });
-
-    describe("storeEvents", function() {
-        var timeline0, i;
-
-        beforeEach(function() {
-            timeline0 = [];
-            for (i = 0; i < batchNum; i++) {
-                timeline0.push(utils.mkMessage({user: userId, room: roomId}));
-            }
-            setItem(mockStorageApi, stateKeyName, {
-                events: stateEventMap,
-                pagination_token: "tok"
-            });
-            setItem(mockStorageApi, prefix + "0", timeline0);
-        });
-
-        it("should add to the live batch", function() {
-            var events = [
-                utils.mkMessage({user: userId, room: roomId, event: true}),
-                utils.mkMessage({user: userId, room: roomId, event: true})
-            ];
-            store.storeEvents(room, events, "atoken");
-            var liveEvents = getItem(mockStorageApi, prefix + "live");
-            expect(liveEvents.length).toEqual(2);
-            expect(liveEvents[0]).toEqual(events[0].event);
-            expect(liveEvents[1]).toEqual(events[1].event);
-        });
-
-        it("should preserve existing live events in the store", function() {
-            var existingEvent = utils.mkMessage({user: userId, room: roomId});
-            setItem(mockStorageApi, prefix + "live", [existingEvent]);
-            var events = [
-                utils.mkMessage({user: userId, room: roomId, event: true}),
-                utils.mkMessage({user: userId, room: roomId, event: true})
-            ];
-            store.storeEvents(room, events, "atoken");
-            var liveEvents = getItem(mockStorageApi, prefix + "live");
-            expect(liveEvents.length).toEqual(3);
-            expect(liveEvents[0]).toEqual(existingEvent);
-            expect(liveEvents[1]).toEqual(events[0].event);
-            expect(liveEvents[2]).toEqual(events[1].event);
-        });
-
-        it("should add to the lowest batch index if toStart=true", function() {
-            var events = [
-                utils.mkMessage({user: userId, room: roomId, event: true}),
-                utils.mkMessage({user: userId, room: roomId, event: true})
-            ];
-            store.storeEvents(room, events, "atoken", true);
-            var timelineNeg1 = getItem(mockStorageApi, prefix + "-1");
-            expect(timelineNeg1.length).toEqual(2);
-            expect(timelineNeg1[0]).toEqual(events[1].event);
-            expect(timelineNeg1[1]).toEqual(events[0].event);
-        });
-
-        it("should add multiple batches to the lowest batch index if toStart=true",
-        function() {
-            var timelineNeg1 = [];
-            var timelineNeg2 = [];
-            for (i = 0; i < batchNum; i++) {
-                timelineNeg1.push(
-                    utils.mkMessage({user: userId, room: roomId, event: true})
-                );
-                timelineNeg2.push(
-                    utils.mkMessage({user: userId, room: roomId, event: true})
-                );
-            }
-
-            var events = timelineNeg2.concat(timelineNeg1).reverse();
-            store.storeEvents(room, events, "atoken", true);
-
-            var storedNeg1 = getItem(mockStorageApi, prefix + "-1");
-            var storedNeg2 = getItem(mockStorageApi, prefix + "-2");
-            expect(timelineNeg1.length).toEqual(storedNeg1.length);
-            expect(timelineNeg2.length).toEqual(storedNeg2.length);
-            for (i = 0; i < timelineNeg1.length; i++) {
-                expect(timelineNeg1[i].event).toEqual(storedNeg1[i]);
-                expect(timelineNeg2[i].event).toEqual(storedNeg2[i]);
-            }
-        });
-
-        it("should update stored state if state events exist", function() {
-            var events = [
-                utils.mkEvent({
-                    user: userId, room: roomId, type: "m.room.name", event: true,
-                    content: {
-                        name: "Room Name Here for updates"
-                    }
-                })
-            ];
-            room.currentState.setStateEvents(events);
-            store.storeEvents(room, events, "atoken");
-
-            var liveEvents = getItem(mockStorageApi, prefix + "live");
-            expect(liveEvents.length).toEqual(1);
-            expect(liveEvents[0]).toEqual(events[0].event);
-
-            var stateEvents = getItem(mockStorageApi, stateKeyName);
-            expect(stateEvents.events["m.room.name"][""]).toEqual(events[0].event);
-        });
-    });
-
-    describe("getRooms", function() {
-        var mkState = function(id) {
-            return [
-                utils.mkEvent({
-                    event: true, type: "m.room.create", user: userId, room: id,
-                    content: {
-                        creator: userId
-                    }
-                }),
-                utils.mkMembership({
-                    event: true, user: userId, room: id, mship: "join"
-                })
-            ];
-        };
-
-        it("should get all rooms in the store", function() {
-            var roomIds = [
-                "!alpha:bet", "!beta:fet"
-            ];
-            // store 2 dynamically
-            var roomA = new Room(roomIds[0]);
-            roomA.currentState.setStateEvents(mkState(roomIds[0]));
-            var roomB = new Room(roomIds[1]);
-            roomB.currentState.setStateEvents(mkState(roomIds[1]));
-            store.storeRoom(roomA);
-            store.storeRoom(roomB);
-
-            var rooms = store.getRooms();
-            expect(rooms.length).toEqual(2);
-            for (var i = 0; i < rooms.length; i++) {
-                var index = roomIds.indexOf(rooms[i].roomId);
-                expect(index).not.toEqual(
-                    -1, "Unknown room"
-                );
-                roomIds.splice(index, 1);
-            }
-        });
-    });
-
-    describe("getUser", function() {
-        it("should be able to retrieve a stored user", function() {
-            var user = new User(userId);
-            store.storeUser(user);
-            var result = store.getUser(userId);
-            expect(result).toBeDefined();
-            expect(result.userId).toEqual(userId);
-        });
-
-        it("should be able to retrieve a stored user with name data", function() {
-            var presence = utils.mkEvent({
-                type: "m.presence", event: true, content: {
-                    user_id: userId,
-                    displayname: "Flibble"
-                }
-            });
-            var user = new User(userId);
-            user.setPresenceEvent(presence);
-            store.storeUser(user);
-            var result = store.getUser(userId);
-            console.log(result);
-            expect(result.events.presence).toEqual(presence);
-        });
-    });
-});
-
-function getItem(store, key) {
-    return JSON.parse(store.getItem(key));
-}
-
-function setItem(store, key, val) {
-    store.setItem(key, JSON.stringify(val));
-}

src/ReEmitter.js

@@ -0,0 +1,46 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module
+ */
+
+export default class Reemitter {
+    constructor(target) {
+        this.target = target;
+
+        // We keep one bound event handler for each event name so we know
+        // what event is arriving
+        this.boundHandlers = {};
+    }
+
+    _handleEvent(eventName, ...args) {
+        this.target.emit(eventName, ...args);
+    }
+
+    reEmit(source, eventNames) {
+        for (const eventName of eventNames) {
+            if (this.boundHandlers[eventName] === undefined) {
+                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
+            }
+            const boundHandler = this.boundHandlers[eventName];
+
+            source.on(eventName, boundHandler);
+        }
+    }
+}

src/content-helpers.js

@@ -0,0 +1,100 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/** @module ContentHelpers */
+module.exports = {
+    /**
+     * Generates the content for a HTML Message event
+     * @param {string} body the plaintext body of the message
+     * @param {string} htmlBody the HTML representation of the message
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlMessage: function(body, htmlBody) {
+        return {
+            msgtype: "m.text",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Notice event
+     * @param {string} body the plaintext body of the notice
+     * @param {string} htmlBody the HTML representation of the notice
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlNotice: function(body, htmlBody) {
+        return {
+            msgtype: "m.notice",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Emote event
+     * @param {string} body the plaintext body of the emote
+     * @param {string} htmlBody the HTML representation of the emote
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlEmote: function(body, htmlBody) {
+        return {
+            msgtype: "m.emote",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Message event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeTextMessage: function(body) {
+        return {
+            msgtype: "m.text",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Notice event
+     * @param {string} body the plaintext body of the notice
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeNotice: function(body) {
+        return {
+            msgtype: "m.notice",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Emote event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeEmoteMessage: function(body) {
+        return {
+            msgtype: "m.emote",
+            body: body,
+        };
+    },
+};

src/content-repo.js

@@ -16,7 +16,7 @@ limitations under the License.
 /**
  * @module content-repo
  */
-var utils = require("./utils");
+const utils = require("./utils");
 
 /** Content Repo utility functions */
 module.exports = {
@@ -46,9 +46,9 @@ module.exports = {
                 return '';
             }
         }
-        var serverAndMediaId = mxc.slice(6); // strips mxc://
-        var prefix = "/_matrix/media/v1/download/";
-        var params = {};
+        let serverAndMediaId = mxc.slice(6); // strips mxc://
+        let prefix = "/_matrix/media/v1/download/";
+        const params = {};
 
         if (width) {
             params.width = width;
@@ -65,8 +65,8 @@ module.exports = {
             prefix = "/_matrix/media/v1/thumbnail/";
         }
 
-        var fragmentOffset = serverAndMediaId.indexOf("#"),
-            fragment = "";
+        const fragmentOffset = serverAndMediaId.indexOf("#");
+        let fragment = "";
         if (fragmentOffset >= 0) {
             fragment = serverAndMediaId.substr(fragmentOffset);
             serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
@@ -88,18 +88,22 @@ module.exports = {
         if (!identiconString) {
             return null;
         }
-        if (!width) { width = 96; }
-        if (!height) { height = 96; }
-        var params = {
+        if (!width) {
+            width = 96;
+        }
+        if (!height) {
+            height = 96;
+        }
+        const params = {
             width: width,
-            height: height
+            height: height,
         };
 
-        var path = utils.encodeUri("/_matrix/media/v1/identicon/$ident", {
-            $ident: identiconString
+        const path = utils.encodeUri("/_matrix/media/v1/identicon/$ident", {
+            $ident: identiconString,
         });
         return baseUrl + path +
             (utils.keys(params).length === 0 ? "" :
                 ("?" + utils.encodeParams(params)));
-    }
+    },
 };

src/crypto/DeviceList.js

@@ -0,0 +1,864 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/**
+ * @module crypto/DeviceList
+ *
+ * Manages the list of other users' devices
+ */
+
+import Promise from 'bluebird';
+
+import logger from '../logger';
+import DeviceInfo from './deviceinfo';
+import olmlib from './olmlib';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+
+/* State transition diagram for DeviceList._deviceTrackingStatus
+ *
+ *                                |
+ *     stopTrackingDeviceList     V
+ *   +---------------------> NOT_TRACKED
+ *   |                            |
+ *   +<--------------------+      | startTrackingDeviceList
+ *   |                     |      V
+ *   |   +-------------> PENDING_DOWNLOAD <--------------------+-+
+ *   |   |                      ^ |                            | |
+ *   |   | restart     download | |  start download            | | invalidateUserDeviceList
+ *   |   | client        failed | |                            | |
+ *   |   |                      | V                            | |
+ *   |   +------------ DOWNLOAD_IN_PROGRESS -------------------+ |
+ *   |                    |       |                              |
+ *   +<-------------------+       |  download successful         |
+ *   ^                            V                              |
+ *   +----------------------- UP_TO_DATE ------------------------+
+ */
+
+
+// constants for DeviceList._deviceTrackingStatus
+const TRACKING_STATUS_NOT_TRACKED = 0;
+const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
+const TRACKING_STATUS_DOWNLOAD_IN_PROGRESS = 2;
+const TRACKING_STATUS_UP_TO_DATE = 3;
+
+/**
+ * @alias module:crypto/DeviceList
+ */
+export default class DeviceList {
+    constructor(baseApis, cryptoStore, sessionStore, olmDevice) {
+        this._cryptoStore = cryptoStore;
+        this._sessionStore = sessionStore;
+
+        // userId -> {
+        //     deviceId -> {
+        //         [device info]
+        //     }
+        // }
+        this._devices = {};
+
+        // which users we are tracking device status for.
+        // userId -> TRACKING_STATUS_*
+        this._deviceTrackingStatus = {}; // loaded from storage in load()
+
+        // The 'next_batch' sync token at the point the data was writen,
+        // ie. a token representing the point immediately after the
+        // moment represented by the snapshot in the db.
+        this._syncToken = null;
+
+        this._serialiser = new DeviceListUpdateSerialiser(
+            baseApis, olmDevice, this,
+        );
+
+        // userId -> promise
+        this._keyDownloadsInProgressByUser = {};
+
+        // Set whenever changes are made other than setting the sync token
+        this._dirty = false;
+
+        // Promise resolved when device data is saved
+        this._savePromise = null;
+        // Function that resolves the save promise
+        this._resolveSavePromise = null;
+        // The time the save is scheduled for
+        this._savePromiseTime = null;
+        // The timer used to delay the save
+        this._saveTimer = null;
+    }
+
+    /**
+     * Load the device tracking state from storage
+     */
+    async load() {
+        let shouldDeleteSessionStore = false;
+        await this._cryptoStore.doTxn(
+            // migrate from session store if there's data there and not here
+            'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    if (deviceData === null) {
+                        logger.log("Migrating e2e device data...");
+                        this._devices = this._sessionStore.getAllEndToEndDevices() || {};
+                        this._deviceTrackingStatus = (
+                            this._sessionStore.getEndToEndDeviceTrackingStatus() || {}
+                        );
+                        this._syncToken = this._sessionStore.getEndToEndDeviceSyncToken();
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                        shouldDeleteSessionStore = true;
+                    } else {
+                        this._devices = deviceData ? deviceData.devices : {},
+                        this._deviceTrackingStatus = deviceData ?
+                            deviceData.trackingStatus : {};
+                        this._syncToken = deviceData ? deviceData.syncToken : null;
+                    }
+                });
+            },
+        );
+
+        if (shouldDeleteSessionStore) {
+            // migrated data is now safely persisted: remove from old store
+            this._sessionStore.removeEndToEndDeviceData();
+        }
+
+        for (const u of Object.keys(this._deviceTrackingStatus)) {
+            // if a download was in progress when we got shut down, it isn't any more.
+            if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
+                this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
+            }
+        }
+    }
+
+    stop() {
+        if (this._saveTimer !== null) {
+            clearTimeout(this._saveTimer);
+        }
+    }
+
+    /**
+     * Save the device tracking state to storage, if any changes are
+     * pending other than updating the sync token
+     *
+     * The actual save will be delayed by a short amount of time to
+     * aggregate multiple writes to the database.
+     *
+     * @param {integer} delay Time in ms before which the save actually happens.
+     *     By default, the save is delayed for a short period in order to batch
+     *     multiple writes, but this behaviour can be disabled by passing 0.
+     *
+     * @return {Promise<bool>} true if the data was saved, false if
+     *     it was not (eg. because no changes were pending). The promise
+     *     will only resolve once the data is saved, so may take some time
+     *     to resolve.
+     */
+    async saveIfDirty(delay) {
+        if (!this._dirty) return Promise.resolve(false);
+        // Delay saves for a bit so we can aggregate multiple saves that happen
+        // in quick succession (eg. when a whole room's devices are marked as known)
+        if (delay === undefined) delay = 500;
+
+        const targetTime = Date.now + delay;
+        if (this._savePromiseTime && targetTime < this._savePromiseTime) {
+            // There's a save scheduled but for after we would like: cancel
+            // it & schedule one for the time we want
+            clearTimeout(this._saveTimer);
+            this._saveTimer = null;
+            this._savePromiseTime = null;
+            // (but keep the save promise since whatever called save before
+            // will still want to know when the save is done)
+        }
+
+        let savePromise = this._savePromise;
+        if (savePromise === null) {
+            savePromise = new Promise((resolve, reject) => {
+                this._resolveSavePromise = resolve;
+            });
+            this._savePromise = savePromise;
+        }
+
+        if (this._saveTimer === null) {
+            const resolveSavePromise = this._resolveSavePromise;
+            this._savePromiseTime = targetTime;
+            this._saveTimer = setTimeout(() => {
+                logger.log('Saving device tracking data at token ' + this._syncToken);
+                // null out savePromise now (after the delay but before the write),
+                // otherwise we could return the existing promise when the save has
+                // actually already happened. Likewise for the dirty flag.
+                this._savePromiseTime = null;
+                this._saveTimer = null;
+                this._savePromise = null;
+                this._resolveSavePromise = null;
+
+                this._dirty = false;
+                this._cryptoStore.doTxn(
+                    'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                    },
+                ).then(() => {
+                    resolveSavePromise();
+                });
+            }, delay);
+        }
+        return savePromise;
+    }
+
+    /**
+     * Gets the sync token last set with setSyncToken
+     *
+     * @return {string} The sync token
+     */
+    getSyncToken() {
+        return this._syncToken;
+    }
+
+    /**
+     * Sets the sync token that the app will pass as the 'since' to the /sync
+     * endpoint next time it syncs.
+     * The sync token must always be set after any changes made as a result of
+     * data in that sync since setting the sync token to a newer one will mean
+     * those changed will not be synced from the server if a new client starts
+     * up with that data.
+     *
+     * @param {string} st The sync token
+     */
+    setSyncToken(st) {
+        this._syncToken = st;
+    }
+
+    /**
+     * Ensures up to date keys for a list of users are stored in the session store,
+     * downloading and storing them if they're not (or if forceDownload is
+     * true).
+     * @param {Array} userIds The users to fetch.
+     * @param {bool} forceDownload Always download the keys even if cached.
+     *
+     * @return {Promise} A promise which resolves to a map userId->deviceId->{@link
+     * module:crypto/deviceinfo|DeviceInfo}.
+     */
+    downloadKeys(userIds, forceDownload) {
+        const usersToDownload = [];
+        const promises = [];
+
+        userIds.forEach((u) => {
+            const trackingStatus = this._deviceTrackingStatus[u];
+            if (this._keyDownloadsInProgressByUser[u]) {
+                // already a key download in progress/queued for this user; its results
+                // will be good enough for us.
+                logger.log(
+                    `downloadKeys: already have a download in progress for ` +
+                    `${u}: awaiting its result`,
+                );
+                promises.push(this._keyDownloadsInProgressByUser[u]);
+            } else if (forceDownload || trackingStatus != TRACKING_STATUS_UP_TO_DATE) {
+                usersToDownload.push(u);
+            }
+        });
+
+        if (usersToDownload.length != 0) {
+            logger.log("downloadKeys: downloading for", usersToDownload);
+            const downloadPromise = this._doKeyDownload(usersToDownload);
+            promises.push(downloadPromise);
+        }
+
+        if (promises.length === 0) {
+            logger.log("downloadKeys: already have all necessary keys");
+        }
+
+        return Promise.all(promises).then(() => {
+            return this._getDevicesFromStore(userIds);
+        });
+    }
+
+    /**
+     * Get the stored device keys for a list of user ids
+     *
+     * @param {string[]} userIds the list of users to list keys for.
+     *
+     * @return {Object} userId->deviceId->{@link module:crypto/deviceinfo|DeviceInfo}.
+     */
+    _getDevicesFromStore(userIds) {
+        const stored = {};
+        const self = this;
+        userIds.map(function(u) {
+            stored[u] = {};
+            const devices = self.getStoredDevicesForUser(u) || [];
+            devices.map(function(dev) {
+                stored[u][dev.deviceId] = dev;
+            });
+        });
+        return stored;
+    }
+
+    /**
+     * Get the stored device keys for a user id
+     *
+     * @param {string} userId the user to list keys for.
+     *
+     * @return {module:crypto/deviceinfo[]|null} list of devices, or null if we haven't
+     * managed to get a list of devices for this user yet.
+     */
+    getStoredDevicesForUser(userId) {
+        const devs = this._devices[userId];
+        if (!devs) {
+            return null;
+        }
+        const res = [];
+        for (const deviceId in devs) {
+            if (devs.hasOwnProperty(deviceId)) {
+                res.push(DeviceInfo.fromStorage(devs[deviceId], deviceId));
+            }
+        }
+        return res;
+    }
+
+    /**
+     * Get the stored device data for a user, in raw object form
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @return {Object} deviceId->{object} devices, or undefined if
+     * there is no data for this user.
+     */
+    getRawStoredDevicesForUser(userId) {
+        return this._devices[userId];
+    }
+
+    /**
+     * Get the stored keys for a single device
+     *
+     * @param {string} userId
+     * @param {string} deviceId
+     *
+     * @return {module:crypto/deviceinfo?} device, or undefined
+     * if we don't know about this device
+     */
+    getStoredDevice(userId, deviceId) {
+        const devs = this._devices[userId];
+        if (!devs || !devs[deviceId]) {
+            return undefined;
+        }
+        return DeviceInfo.fromStorage(devs[deviceId], deviceId);
+    }
+
+    /**
+     * Find a device by curve25519 identity key
+     *
+     * @param {string} userId     owner of the device
+     * @param {string} algorithm  encryption algorithm
+     * @param {string} senderKey  curve25519 key to match
+     *
+     * @return {module:crypto/deviceinfo?}
+     */
+    getDeviceByIdentityKey(userId, algorithm, senderKey) {
+        if (
+            algorithm !== olmlib.OLM_ALGORITHM &&
+            algorithm !== olmlib.MEGOLM_ALGORITHM
+        ) {
+            // we only deal in olm keys
+            return null;
+        }
+
+        const devices = this._devices[userId];
+        if (!devices) {
+            return null;
+        }
+
+        for (const deviceId in devices) {
+            if (!devices.hasOwnProperty(deviceId)) {
+                continue;
+            }
+
+            const device = devices[deviceId];
+            for (const keyId in device.keys) {
+                if (!device.keys.hasOwnProperty(keyId)) {
+                    continue;
+                }
+                if (keyId.indexOf("curve25519:") !== 0) {
+                    continue;
+                }
+                const deviceKey = device.keys[keyId];
+                if (deviceKey == senderKey) {
+                    return DeviceInfo.fromStorage(device, deviceId);
+                }
+            }
+        }
+
+        // doesn't match a known device
+        return null;
+    }
+
+    /**
+     * Replaces the list of devices for a user with the given device list
+     *
+     * @param {string} u The user ID
+     * @param {Object} devs New device info for user
+     */
+    storeDevicesForUser(u, devs) {
+        this._devices[u] = devs;
+        this._dirty = true;
+    }
+
+    /**
+     * flag the given user for device-list tracking, if they are not already.
+     *
+     * This will mean that a subsequent call to refreshOutdatedDeviceLists()
+     * will download the device list for the user, and that subsequent calls to
+     * invalidateUserDeviceList will trigger more updates.
+     *
+     * @param {String} userId
+     */
+    startTrackingDeviceList(userId) {
+        // sanity-check the userId. This is mostly paranoia, but if synapse
+        // can't parse the userId we give it as an mxid, it 500s the whole
+        // request and we can never update the device lists again (because
+        // the broken userId is always 'invalid' and always included in any
+        // refresh request).
+        // By checking it is at least a string, we can eliminate a class of
+        // silly errors.
+        if (typeof userId !== 'string') {
+            throw new Error('userId must be a string; was '+userId);
+        }
+        if (!this._deviceTrackingStatus[userId]) {
+            logger.log('Now tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+        }
+        // we don't yet persist the tracking status, since there may be a lot
+        // of calls; we save all data together once the sync is done
+        this._dirty = true;
+    }
+
+    /**
+     * Mark the given user as no longer being tracked for device-list updates.
+     *
+     * This won't affect any in-progress downloads, which will still go on to
+     * complete; it will just mean that we don't think that we have an up-to-date
+     * list for future calls to downloadKeys.
+     *
+     * @param {String} userId
+     */
+    stopTrackingDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            logger.log('No longer tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
+        }
+    }
+
+    /**
+     * Set all users we're currently tracking to untracked
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    stopTrackingAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        this._dirty = true;
+    }
+
+    /**
+     * Mark the cached device list for the given user outdated.
+     *
+     * If we are not tracking this user's devices, we'll do nothing. Otherwise
+     * we flag the user as needing an update.
+     *
+     * This doesn't actually set off an update, so that several users can be
+     * batched together. Call refreshOutdatedDeviceLists() for that.
+     *
+     * @param {String} userId
+     */
+    invalidateUserDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            logger.log("Marking device list outdated for", userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
+        }
+    }
+
+    /**
+     * If we have users who have outdated device lists, start key downloads for them
+     *
+     * @returns {Promise} which completes when the download completes; normally there
+     *    is no need to wait for this (it's mostly for the unit tests).
+     */
+    refreshOutdatedDeviceLists() {
+        this.saveIfDirty();
+
+        const usersToDownload = [];
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            const stat = this._deviceTrackingStatus[userId];
+            if (stat == TRACKING_STATUS_PENDING_DOWNLOAD) {
+                usersToDownload.push(userId);
+            }
+        }
+
+        return this._doKeyDownload(usersToDownload);
+    }
+
+    /**
+     * Set the stored device data for a user, in raw object form
+     * Used only by internal class DeviceListUpdateSerialiser
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @param {Object} devices deviceId->{object} the new devices
+     */
+    _setRawStoredDevicesForUser(userId, devices) {
+        this._devices[userId] = devices;
+    }
+
+    /**
+     * Fire off download update requests for the given users, and update the
+     * device list tracking status for them, and the
+     * _keyDownloadsInProgressByUser map for them.
+     *
+     * @param {String[]} users  list of userIds
+     *
+     * @return {module:client.Promise} resolves when all the users listed have
+     *     been updated. rejects if there was a problem updating any of the
+     *     users.
+     */
+    _doKeyDownload(users) {
+        if (users.length === 0) {
+            // nothing to do
+            return Promise.resolve();
+        }
+
+        const prom = this._serialiser.updateDevicesForUsers(
+            users, this._syncToken,
+        ).then(() => {
+            finished(true);
+        }, (e) => {
+            logger.error(
+                'Error downloading keys for ' + users + ":", e,
+            );
+            finished(false);
+            throw e;
+        });
+
+        users.forEach((u) => {
+            this._keyDownloadsInProgressByUser[u] = prom;
+            const stat = this._deviceTrackingStatus[u];
+            if (stat == TRACKING_STATUS_PENDING_DOWNLOAD) {
+                this._deviceTrackingStatus[u] = TRACKING_STATUS_DOWNLOAD_IN_PROGRESS;
+            }
+        });
+
+        const finished = (success) => {
+            users.forEach((u) => {
+                this._dirty = true;
+
+                // we may have queued up another download request for this user
+                // since we started this request. If that happens, we should
+                // ignore the completion of the first one.
+                if (this._keyDownloadsInProgressByUser[u] !== prom) {
+                    logger.log('Another update in the queue for', u,
+                                '- not marking up-to-date');
+                    return;
+                }
+                delete this._keyDownloadsInProgressByUser[u];
+                const stat = this._deviceTrackingStatus[u];
+                if (stat == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
+                    if (success) {
+                        // we didn't get any new invalidations since this download started:
+                        // this user's device list is now up to date.
+                        this._deviceTrackingStatus[u] = TRACKING_STATUS_UP_TO_DATE;
+                        logger.log("Device list for", u, "now up to date");
+                    } else {
+                        this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
+                    }
+                }
+            });
+            this.saveIfDirty();
+        };
+
+        return prom;
+    }
+}
+
+/**
+ * Serialises updates to device lists
+ *
+ * Ensures that results from /keys/query are not overwritten if a second call
+ * completes *before* an earlier one.
+ *
+ * It currently does this by ensuring only one call to /keys/query happens at a
+ * time (and queuing other requests up).
+ */
+class DeviceListUpdateSerialiser {
+    /*
+     * @param {object} baseApis Base API object
+     * @param {object} olmDevice The Olm Device
+     * @param {object} deviceList The device list object
+     */
+    constructor(baseApis, olmDevice, deviceList) {
+        this._baseApis = baseApis;
+        this._olmDevice = olmDevice;
+        this._deviceList = deviceList; // the device list to be updated
+
+        this._downloadInProgress = false;
+
+        // users which are queued for download
+        // userId -> true
+        this._keyDownloadsQueuedByUser = {};
+
+        // deferred which is resolved when the queued users are downloaded.
+        //
+        // non-null indicates that we have users queued for download.
+        this._queuedQueryDeferred = null;
+
+        this._syncToken = null; // The sync token we send with the requests
+    }
+
+    /**
+     * Make a key query request for the given users
+     *
+     * @param {String[]} users list of user ids
+     *
+     * @param {String} syncToken sync token to pass in the query request, to
+     *     help the HS give the most recent results
+     *
+     * @return {module:client.Promise} resolves when all the users listed have
+     *     been updated. rejects if there was a problem updating any of the
+     *     users.
+     */
+    updateDevicesForUsers(users, syncToken) {
+        users.forEach((u) => {
+            this._keyDownloadsQueuedByUser[u] = true;
+        });
+
+        if (!this._queuedQueryDeferred) {
+            this._queuedQueryDeferred = Promise.defer();
+        }
+
+        // We always take the new sync token and just use the latest one we've
+        // been given, since it just needs to be at least as recent as the
+        // sync response the device invalidation message arrived in
+        this._syncToken = syncToken;
+
+        if (this._downloadInProgress) {
+            // just queue up these users
+            logger.log('Queued key download for', users);
+            return this._queuedQueryDeferred.promise;
+        }
+
+        // start a new download.
+        return this._doQueuedQueries();
+    }
+
+    _doQueuedQueries() {
+        if (this._downloadInProgress) {
+            throw new Error(
+                "DeviceListUpdateSerialiser._doQueuedQueries called with request active",
+            );
+        }
+
+        const downloadUsers = Object.keys(this._keyDownloadsQueuedByUser);
+        this._keyDownloadsQueuedByUser = {};
+        const deferred = this._queuedQueryDeferred;
+        this._queuedQueryDeferred = null;
+
+        logger.log('Starting key download for', downloadUsers);
+        this._downloadInProgress = true;
+
+        const opts = {};
+        if (this._syncToken) {
+            opts.token = this._syncToken;
+        }
+
+        this._baseApis.downloadKeysForUsers(
+            downloadUsers, opts,
+        ).then((res) => {
+            const dk = res.device_keys || {};
+
+            // do each user in a separate promise, to avoid wedging the CPU
+            // (https://github.com/vector-im/riot-web/issues/3158)
+            //
+            // of course we ought to do this in a web worker or similar, but
+            // this serves as an easy solution for now.
+            let prom = Promise.resolve();
+            for (const userId of downloadUsers) {
+                prom = prom.delay(5).then(() => {
+                    return this._processQueryResponseForUser(userId, dk[userId]);
+                });
+            }
+
+            return prom;
+        }).done(() => {
+            logger.log('Completed key download for ' + downloadUsers);
+
+            this._downloadInProgress = false;
+            deferred.resolve();
+
+            // if we have queued users, fire off another request.
+            if (this._queuedQueryDeferred) {
+                this._doQueuedQueries();
+            }
+        }, (e) => {
+            logger.warn('Error downloading keys for ' + downloadUsers + ':', e);
+            this._downloadInProgress = false;
+            deferred.reject(e);
+        });
+
+        return deferred.promise;
+    }
+
+    async _processQueryResponseForUser(userId, response) {
+        logger.log('got keys for ' + userId + ':', response);
+
+        // map from deviceid -> deviceinfo for this user
+        const userStore = {};
+        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
+        if (devs) {
+            Object.keys(devs).forEach((deviceId) => {
+                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                userStore[deviceId] = d;
+            });
+        }
+
+        await _updateStoredDeviceKeysForUser(
+            this._olmDevice, userId, userStore, response || {},
+        );
+
+        // put the updates into thr object that will be returned as our results
+        const storage = {};
+        Object.keys(userStore).forEach((deviceId) => {
+            storage[deviceId] = userStore[deviceId].toStorage();
+        });
+
+        this._deviceList._setRawStoredDevicesForUser(userId, storage);
+    }
+}
+
+
+async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
+        userResult) {
+    let updated = false;
+
+    // remove any devices in the store which aren't in the response
+    for (const deviceId in userStore) {
+        if (!userStore.hasOwnProperty(deviceId)) {
+            continue;
+        }
+
+        if (!(deviceId in userResult)) {
+            logger.log("Device " + userId + ":" + deviceId +
+                " has been removed");
+            delete userStore[deviceId];
+            updated = true;
+        }
+    }
+
+    for (const deviceId in userResult) {
+        if (!userResult.hasOwnProperty(deviceId)) {
+            continue;
+        }
+
+        const deviceResult = userResult[deviceId];
+
+        // check that the user_id and device_id in the response object are
+        // correct
+        if (deviceResult.user_id !== userId) {
+            logger.warn("Mismatched user_id " + deviceResult.user_id +
+               " in keys from " + userId + ":" + deviceId);
+            continue;
+        }
+        if (deviceResult.device_id !== deviceId) {
+            logger.warn("Mismatched device_id " + deviceResult.device_id +
+               " in keys from " + userId + ":" + deviceId);
+            continue;
+        }
+
+        if (await _storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
+            updated = true;
+        }
+    }
+
+    return updated;
+}
+
+/*
+ * Process a device in a /query response, and add it to the userStore
+ *
+ * returns (a promise for) true if a change was made, else false
+ */
+async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
+    if (!deviceResult.keys) {
+        // no keys?
+        return false;
+    }
+
+    const deviceId = deviceResult.device_id;
+    const userId = deviceResult.user_id;
+
+    const signKeyId = "ed25519:" + deviceId;
+    const signKey = deviceResult.keys[signKeyId];
+    if (!signKey) {
+        logger.warn("Device " + userId + ":" + deviceId +
+            " has no ed25519 key");
+        return false;
+    }
+
+    const unsigned = deviceResult.unsigned || {};
+
+    try {
+        await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
+    } catch (e) {
+        logger.warn("Unable to verify signature on device " +
+            userId + ":" + deviceId + ":" + e);
+        return false;
+    }
+
+    // DeviceInfo
+    let deviceStore;
+
+    if (deviceId in userStore) {
+        // already have this device.
+        deviceStore = userStore[deviceId];
+
+        if (deviceStore.getFingerprint() != signKey) {
+            // this should only happen if the list has been MITMed; we are
+            // best off sticking with the original keys.
+            //
+            // Should we warn the user about it somehow?
+            logger.warn("Ed25519 key for device " + userId + ":" +
+               deviceId + " has changed");
+            return false;
+        }
+    } else {
+        userStore[deviceId] = deviceStore = new DeviceInfo(deviceId);
+    }
+
+    deviceStore.keys = deviceResult.keys || {};
+    deviceStore.algorithms = deviceResult.algorithms || [];
+    deviceStore.unsigned = unsigned;
+    return true;
+}

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -0,0 +1,408 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../logger';
+import utils from '../utils';
+
+/**
+ * Internal module. Management of outgoing room key requests.
+ *
+ * See https://docs.google.com/document/d/1m4gQkcnJkxNuBmb5NoFCIadIY-DyqqNAS3lloE73BlQ
+ * for draft documentation on what we're supposed to be implementing here.
+ *
+ * @module
+ */
+
+// delay between deciding we want some keys, and sending out the request, to
+// allow for (a) it turning up anyway, (b) grouping requests together
+const SEND_KEY_REQUESTS_DELAY_MS = 500;
+
+/** possible states for a room key request
+ *
+ * The state machine looks like:
+ *
+ *     |         (cancellation sent)
+ *     | .-------------------------------------------------.
+ *     | |                                                 |
+ *     V V       (cancellation requested)                  |
+ *   UNSENT  -----------------------------+                |
+ *     |                                  |                |
+ *     |                                  |                |
+ *     | (send successful)                |  CANCELLATION_PENDING_AND_WILL_RESEND
+ *     V                                  |                Λ
+ *    SENT                                |                |
+ *     |--------------------------------  |  --------------'
+ *     |                                  |  (cancellation requested with intent
+ *     |                                  |   to resend the original request)
+ *     |                                  |
+ *     | (cancellation requested)         |
+ *     V                                  |
+ * CANCELLATION_PENDING                   |
+ *     |                                  |
+ *     | (cancellation sent)              |
+ *     V                                  |
+ * (deleted)  <---------------------------+
+ *
+ * @enum {number}
+ */
+const ROOM_KEY_REQUEST_STATES = {
+    /** request not yet sent */
+    UNSENT: 0,
+
+    /** request sent, awaiting reply */
+    SENT: 1,
+
+    /** reply received, cancellation not yet sent */
+    CANCELLATION_PENDING: 2,
+
+    /**
+     * Cancellation not yet sent and will transition to UNSENT instead of
+     * being deleted once the cancellation has been sent.
+     */
+    CANCELLATION_PENDING_AND_WILL_RESEND: 3,
+};
+
+export default class OutgoingRoomKeyRequestManager {
+    constructor(baseApis, deviceId, cryptoStore) {
+        this._baseApis = baseApis;
+        this._deviceId = deviceId;
+        this._cryptoStore = cryptoStore;
+
+        // handle for the delayed call to _sendOutgoingRoomKeyRequests. Non-null
+        // if the callback has been set, or if it is still running.
+        this._sendOutgoingRoomKeyRequestsTimer = null;
+
+        // sanity check to ensure that we don't end up with two concurrent runs
+        // of _sendOutgoingRoomKeyRequests
+        this._sendOutgoingRoomKeyRequestsRunning = false;
+
+        this._clientRunning = false;
+    }
+
+    /**
+     * Called when the client is started. Sets background processes running.
+     */
+    start() {
+        this._clientRunning = true;
+
+        // set the timer going, to handle any requests which didn't get sent
+        // on the previous run of the client.
+        this._startTimer();
+    }
+
+    /**
+     * Called when the client is stopped. Stops any running background processes.
+     */
+    stop() {
+        logger.log('stopping OutgoingRoomKeyRequestManager');
+        // stop the timer on the next run
+        this._clientRunning = false;
+    }
+
+    /**
+     * Send off a room key request, if we haven't already done so.
+     *
+     * The `requestBody` is compared (with a deep-equality check) against
+     * previous queued or sent requests and if it matches, no change is made.
+     * Otherwise, a request is added to the pending list, and a job is started
+     * in the background to send it.
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {Array<{userId: string, deviceId: string}>} recipients
+     *
+     * @returns {Promise} resolves when the request has been added to the
+     *    pending list (or we have established that a similar request already
+     *    exists)
+     */
+    sendRoomKeyRequest(requestBody, recipients) {
+        return this._cryptoStore.getOrAddOutgoingRoomKeyRequest({
+            requestBody: requestBody,
+            recipients: recipients,
+            requestId: this._baseApis.makeTxnId(),
+            state: ROOM_KEY_REQUEST_STATES.UNSENT,
+        }).then((req) => {
+            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
+                this._startTimer();
+            }
+        });
+    }
+
+    /**
+     * Cancel room key requests, if any match the given requestBody
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {boolean} andResend if true, transition to UNSENT instead of
+     *                            deleting after sending cancellation.
+     *
+     * @returns {Promise} resolves when the request has been updated in our
+     *    pending list.
+     */
+    cancelRoomKeyRequest(requestBody, andResend=false) {
+        return this._cryptoStore.getOutgoingRoomKeyRequest(
+            requestBody,
+        ).then((req) => {
+            if (!req) {
+                // no request was made for this key
+                return;
+            }
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    // nothing to do here
+                    return;
+
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    // just delete it
+
+                    // FIXME: ghahah we may have attempted to send it, and
+                    // not yet got a successful response. So the server
+                    // may have seen it, so we still need to send a cancellation
+                    // in that case :/
+
+                    logger.log(
+                        'deleting unnecessary room key request for ' +
+                        stringifyRequestBody(requestBody),
+                    );
+                    return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                    );
+
+                case ROOM_KEY_REQUEST_STATES.SENT: {
+                    // If `andResend` is set, transition to UNSENT once the
+                    // cancellation has successfully been sent.
+                    const state = andResend ?
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING;
+                    // send a cancellation.
+                    return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
+                            state,
+                            cancellationTxnId: this._baseApis.makeTxnId(),
+                        },
+                    ).then((updatedReq) => {
+                        if (!updatedReq) {
+                            // updateOutgoingRoomKeyRequest couldn't find the
+                            // request in state ROOM_KEY_REQUEST_STATES.SENT,
+                            // so we must have raced with another tab to mark
+                            // the request cancelled. There is no point in
+                            // sending another cancellation since the other tab
+                            // will do it.
+                            logger.log(
+                                'Tried to cancel room key request for ' +
+                                stringifyRequestBody(requestBody) +
+                                ' but it was already cancelled in another tab',
+                            );
+                            return;
+                        }
+
+                        // We don't want to wait for the timer, so we send it
+                        // immediately. (We might actually end up racing with the timer,
+                        // but that's ok: even if we make the request twice, we'll do it
+                        // with the same transaction_id, so only one message will get
+                        // sent).
+                        //
+                        // (We also don't want to wait for the response from the server
+                        // here, as it will slow down processing of received keys if we
+                        // do.)
+                        this._sendOutgoingRoomKeyRequestCancellation(
+                            updatedReq,
+                            andResend,
+                        ).catch((e) => {
+                            logger.error(
+                                "Error sending room key request cancellation;"
+                                + " will retry later.", e,
+                            );
+                            this._startTimer();
+                        }).then(() => {
+                            if (!andResend) return;
+                            // The request has transitioned from
+                            // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
+                            // still need to resend the request which is now UNSENT, so
+                            // start the timer if it isn't already started.
+                            this._startTimer();
+                        });
+                    });
+                }
+                default:
+                    throw new Error('unhandled state: ' + req.state);
+            }
+        });
+    }
+
+    // start the background timer to send queued requests, if the timer isn't
+    // already running
+    _startTimer() {
+        if (this._sendOutgoingRoomKeyRequestsTimer) {
+            return;
+        }
+
+        const startSendingOutgoingRoomKeyRequests = () => {
+            if (this._sendOutgoingRoomKeyRequestsRunning) {
+                throw new Error("RoomKeyRequestSend already in progress!");
+            }
+            this._sendOutgoingRoomKeyRequestsRunning = true;
+
+            this._sendOutgoingRoomKeyRequests().finally(() => {
+                this._sendOutgoingRoomKeyRequestsRunning = false;
+            }).catch((e) => {
+                // this should only happen if there is an indexeddb error,
+                // in which case we're a bit stuffed anyway.
+                logger.warn(
+                    `error in OutgoingRoomKeyRequestManager: ${e}`,
+                );
+            }).done();
+        };
+
+        this._sendOutgoingRoomKeyRequestsTimer = global.setTimeout(
+            startSendingOutgoingRoomKeyRequests,
+            SEND_KEY_REQUESTS_DELAY_MS,
+        );
+    }
+
+    // look for and send any queued requests. Runs itself recursively until
+    // there are no more requests, or there is an error (in which case, the
+    // timer will be restarted before the promise resolves).
+    _sendOutgoingRoomKeyRequests() {
+        if (!this._clientRunning) {
+            this._sendOutgoingRoomKeyRequestsTimer = null;
+            return Promise.resolve();
+        }
+
+        logger.log("Looking for queued outgoing room key requests");
+
+        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+            ROOM_KEY_REQUEST_STATES.UNSENT,
+        ]).then((req) => {
+            if (!req) {
+                logger.log("No more outgoing room key requests");
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                return;
+            }
+
+            let prom;
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    prom = this._sendOutgoingRoomKeyRequest(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req, true);
+                    break;
+            }
+
+            return prom.then(() => {
+                // go around the loop again
+                return this._sendOutgoingRoomKeyRequests();
+            }).catch((e) => {
+                logger.error("Error sending room key request; will retry later.", e);
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                this._startTimer();
+            }).done();
+        });
+    }
+
+    // given a RoomKeyRequest, send it and update the request record
+    _sendOutgoingRoomKeyRequest(req) {
+        logger.log(
+            `Requesting keys for ${stringifyRequestBody(req.requestBody)}` +
+            ` from ${stringifyRecipientList(req.recipients)}` +
+            `(id ${req.requestId})`,
+        );
+
+        const requestMessage = {
+            action: "request",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+            body: req.requestBody,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.requestId,
+        ).then(() => {
+            return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                { state: ROOM_KEY_REQUEST_STATES.SENT },
+            );
+        });
+    }
+
+    // Given a RoomKeyRequest, cancel it and delete the request record unless
+    // andResend is set, in which case transition to UNSENT.
+    _sendOutgoingRoomKeyRequestCancellation(req, andResend) {
+        logger.log(
+            `Sending cancellation for key request for ` +
+            `${stringifyRequestBody(req.requestBody)} to ` +
+            `${stringifyRecipientList(req.recipients)} ` +
+            `(cancellation id ${req.cancellationTxnId})`,
+        );
+
+        const requestMessage = {
+            action: "request_cancellation",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.cancellationTxnId,
+        ).then(() => {
+            if (andResend) {
+                // We want to resend, so transition to UNSENT
+                return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId,
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+                    { state: ROOM_KEY_REQUEST_STATES.UNSENT },
+                );
+            }
+            return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            );
+        });
+    }
+
+    // send a RoomKeyRequest to a list of recipients
+    _sendMessageToDevices(message, recipients, txnId) {
+        const contentMap = {};
+        for (const recip of recipients) {
+            if (!contentMap[recip.userId]) {
+                contentMap[recip.userId] = {};
+            }
+            contentMap[recip.userId][recip.deviceId] = message;
+        }
+
+        return this._baseApis.sendToDevice(
+            'm.room_key_request', contentMap, txnId,
+        );
+    }
+}
+
+function stringifyRequestBody(requestBody) {
+    // we assume that the request is for megolm keys, which are identified by
+    // room id and session id
+    return requestBody.room_id + " / " + requestBody.session_id;
+}
+
+function stringifyRecipientList(recipients) {
+    return '['
+        + utils.map(recipients, (r) => `${r.userId}:${r.deviceId}`).join(",")
+        + ']';
+}
+

src/crypto/RoomList.js

@@ -0,0 +1,84 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module crypto/RoomList
+ *
+ * Manages the list of encrypted rooms
+ */
+
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+/**
+ * @alias module:crypto/RoomList
+ */
+export default class RoomList {
+    constructor(cryptoStore, sessionStore) {
+        this._cryptoStore = cryptoStore;
+        this._sessionStore = sessionStore;
+
+        // Object of roomId -> room e2e info object (body of the m.room.encryption event)
+        this._roomEncryption = {};
+    }
+
+    async init() {
+        let removeSessionStoreRooms = false;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.getEndToEndRooms(txn, (result) => {
+                    if (result === null || Object.keys(result).length === 0) {
+                        // migrate from session store, if there's data there
+                        const sessStoreRooms = this._sessionStore.getAllEndToEndRooms();
+                        if (sessStoreRooms !== null) {
+                            for (const roomId of Object.keys(sessStoreRooms)) {
+                                this._cryptoStore.storeEndToEndRoom(
+                                    roomId, sessStoreRooms[roomId], txn,
+                                );
+                            }
+                        }
+                        this._roomEncryption = sessStoreRooms;
+                        removeSessionStoreRooms = true;
+                    } else {
+                        this._roomEncryption = result;
+                    }
+                });
+            },
+        );
+        if (removeSessionStoreRooms) {
+            this._sessionStore.removeAllEndToEndRooms();
+        }
+    }
+
+    getRoomEncryption(roomId) {
+        return this._roomEncryption[roomId] || null;
+    }
+
+    isRoomEncrypted(roomId) {
+        return Boolean(this.getRoomEncryption(roomId));
+    }
+
+    async setRoomEncryption(roomId, roomInfo) {
+        // important that this happens before calling into the store
+        // as it prevents the Crypto::setRoomEncryption from calling
+        // this twice for consecutive m.room.encryption events
+        this._roomEncryption[roomId] = roomInfo;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+            },
+        );
+    }
+}

src/crypto/algorithms/base.js

@@ -13,14 +13,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * Internal module. Defines the base classes of the encryption implementations
  *
- * @module crypto/algorithms/base
+ * @module
  */
-var utils = require("../../utils");
+
+import Promise from 'bluebird';
 
 /**
  * map of registered encryption algorithm classes. A map from string to {@link
@@ -28,7 +28,7 @@ var utils = require("../../utils");
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.EncryptionAlgorithm)>}
  */
-module.exports.ENCRYPTION_CLASSES = {};
+export const ENCRYPTION_CLASSES = {};
 
 /**
  * map of registered encryption algorithm classes. Map from string to {@link
@@ -36,12 +36,11 @@ module.exports.ENCRYPTION_CLASSES = {};
  *
  * @type {Object.<string, function(new: module:crypto/algorithms/base.DecryptionAlgorithm)>}
  */
-module.exports.DECRYPTION_CLASSES = {};
+export const DECRYPTION_CLASSES = {};
 
 /**
  * base type for encryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.EncryptionAlgorithm
  *
  * @param {object} params parameters
@@ -53,100 +52,170 @@ module.exports.DECRYPTION_CLASSES = {};
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
-var EncryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._deviceId = params.deviceId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._baseApis = params.baseApis;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.EncryptionAlgorithm = EncryptionAlgorithm;
+class EncryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._deviceId = params.deviceId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Encrypt a message event
- *
- * @method module:crypto/algorithms/base.EncryptionAlgorithm#encryptMessage
- * @abstract
- *
- * @param {module:models/room} room
- * @param {string} eventType
- * @param {object} plaintext event content
- *
- * @return {module:client.Promise} Promise which resolves to the new event body
- */
+    /**
+     * Encrypt a message event
+     *
+     * @method module:crypto/algorithms/base.EncryptionAlgorithm.encryptMessage
+     * @abstract
+     *
+     * @param {module:models/room} room
+     * @param {string} eventType
+     * @param {object} plaintext event content
+     *
+     * @return {module:client.Promise} Promise which resolves to the new event body
+     */
 
-/**
- * Called when the membership of a member of the room changes.
- *
- * @param {module:models/event.MatrixEvent} event  event causing the change
- * @param {module:models/room-member} member  user whose membership changed
- * @param {string=} oldMembership  previous membership
- */
-EncryptionAlgorithm.prototype.onRoomMembership = function(
-    event, member, oldMembership
-) {};
+    /**
+     * Called when the membership of a member of the room changes.
+     *
+     * @param {module:models/event.MatrixEvent} event  event causing the change
+     * @param {module:models/room-member} member  user whose membership changed
+     * @param {string=} oldMembership  previous membership
+     * @public
+     */
+     onRoomMembership(event, member, oldMembership) {
+     }
+}
+export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * base type for decryption implementations
  *
- * @constructor
  * @alias module:crypto/algorithms/base.DecryptionAlgorithm
- *
  * @param {object} params parameters
  * @param {string} params.userId  The UserID for the local user
  * @param {module:crypto} params.crypto crypto core
  * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
+ * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */
-var DecryptionAlgorithm = function(params) {
-    this._userId = params.userId;
-    this._crypto = params.crypto;
-    this._olmDevice = params.olmDevice;
-    this._roomId = params.roomId;
-};
-/** */
-module.exports.DecryptionAlgorithm = DecryptionAlgorithm;
+class DecryptionAlgorithm {
+    constructor(params) {
+        this._userId = params.userId;
+        this._crypto = params.crypto;
+        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
+        this._roomId = params.roomId;
+    }
 
-/**
- * Decrypt an event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
- * @abstract
- *
- * @param {object} event raw event
- *
- * @return {null} if the event referred to an unknown megolm session
- * @return {module:crypto.DecryptionResult} decryption result
- *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
- */
+    /**
+     * Decrypt an event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
+     * @abstract
+     *
+     * @param {MatrixEvent} event undecrypted event
+     *
+     * @return {Promise<module:crypto~EventDecryptionResult>} promise which
+     * resolves once we have finished decrypting. Rejects with an
+     * `algorithms.DecryptionError` if there is a problem decrypting the event.
+     */
 
-/**
- * Handle a key event
- *
- * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
- *
- * @param {module:models/event.MatrixEvent} params event key event
- */
-DecryptionAlgorithm.prototype.onRoomKeyEvent = function(params) {
-    // ignore by default
-};
+    /**
+     * Handle a key event
+     *
+     * @method module:crypto/algorithms/base.DecryptionAlgorithm#onRoomKeyEvent
+     *
+     * @param {module:models/event.MatrixEvent} params event key event
+     */
+    onRoomKeyEvent(params) {
+        // ignore by default
+    }
+
+    /**
+     * Import a room key
+     *
+     * @param {module:crypto/OlmDevice.MegolmSessionData} session
+     */
+    importRoomKey(session) {
+        // ignore by default
+    }
+
+    /**
+     * Determine if we have the keys necessary to respond to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     * @return {Promise<boolean>} true if we have the keys and could (theoretically) share
+     *  them; else false.
+     */
+    hasKeysForKeyRequest(keyRequest) {
+        return Promise.resolve(false);
+    }
+
+    /**
+     * Send the response to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     */
+    shareKeysWithDevice(keyRequest) {
+        throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
+    }
+}
+export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown when decryption fails
  *
- * @constructor
- * @param {string} msg message describing the problem
+ * @alias module:crypto/algorithms/base.DecryptionError
+ * @param {string} msg user-visible message describing the problem
+ *
+ * @param {Object=} details key/value pairs reported in the logs but not shown
+ *   to the user.
+ *
  * @extends Error
  */
-module.exports.DecryptionError = function(msg) {
-    this.message = msg;
-};
-utils.inherits(module.exports.DecryptionError, Error);
+class DecryptionError extends Error {
+    constructor(code, msg, details) {
+        super(msg);
+        this.code = code;
+        this.name = 'DecryptionError';
+        this.detailedString = _detailedStringForDecryptionError(this, details);
+    }
+}
+export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
+
+function _detailedStringForDecryptionError(err, details) {
+    let result = err.name + '[msg: ' + err.message;
+
+    if (details) {
+        result += ', ' +
+            Object.keys(details).map(
+                (k) => k + ': ' + details[k],
+            ).join(', ');
+    }
+
+    result += ']';
+
+    return result;
+}
+
+/**
+ * Exception thrown specifically when we want to warn the user to consider
+ * the security of their conversation before continuing
+ *
+ * @param {string} msg message describing the problem
+ * @param {Object} devices userId -> {deviceId -> object}
+ *      set of unknown devices per user we're warning about
+ * @extends Error
+ */
+export class UnknownDeviceError extends Error {
+    constructor(msg, devices) {
+        super(msg);
+        this.name = "UnknownDeviceError";
+        this.devices = devices;
+    }
+}
 
 /**
  * Registers an encryption/decryption class for a particular algorithm
@@ -161,7 +230,7 @@ utils.inherits(module.exports.DecryptionError, Error);
  *     module:crypto/algorithms/base.DecryptionAlgorithm|DecryptionAlgorithm}
  *     implementation
  */
-module.exports.registerAlgorithm = function(algorithm, encryptor, decryptor) {
-    module.exports.ENCRYPTION_CLASSES[algorithm] = encryptor;
-    module.exports.DECRYPTION_CLASSES[algorithm] = decryptor;
-};
+export function registerAlgorithm(algorithm, encryptor, decryptor) {
+    ENCRYPTION_CLASSES[algorithm] = encryptor;
+    DECRYPTION_CLASSES[algorithm] = decryptor;
+}

src/crypto/algorithms/index.js

@@ -19,7 +19,7 @@ limitations under the License.
  * @module crypto/algorithms
  */
 
-var base = require("./base");
+const base = require("./base");
 
 require("./olm");
 require("./megolm");

src/crypto/algorithms/olm.js

@@ -20,15 +20,16 @@ limitations under the License.
  *
  * @module crypto/algorithms/olm
  */
-var q = require('q');
+import Promise from 'bluebird';
 
-var utils = require("../../utils");
-var olmlib = require("../olmlib");
-var DeviceInfo = require("../deviceinfo");
-var DeviceVerification = DeviceInfo.DeviceVerification;
+const logger = require("../../logger");
+const utils = require("../../utils");
+const olmlib = require("../olmlib");
+const DeviceInfo = require("../deviceinfo");
+const DeviceVerification = DeviceInfo.DeviceVerification;
 
 
-var base = require("./base");
+const base = require("./base");
 
 /**
  * Olm encryption implementation
@@ -60,11 +61,11 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
 
     if (this._sessionPrepared) {
         // prep already done
-        return q();
+        return Promise.resolve();
     }
 
-    var self = this;
-    this._prepPromise = self._crypto.downloadKeys(roomMembers, true).then(function(res) {
+    const self = this;
+    this._prepPromise = self._crypto.downloadKeys(roomMembers).then(function(res) {
         return self._crypto.ensureOlmSessionsForUsers(roomMembers);
     }).then(function() {
         self._sessionPrepared = true;
@@ -83,56 +84,62 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
  *
  * @return {module:client.Promise} Promise which resolves to the new event body
  */
-OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {
+OlmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
     // pick the list of recipients based on the membership list.
     //
     // TODO: there is a race condition here! What if a new user turns up
     // just as you are sending a secret message?
 
-    var users = utils.map(room.getJoinedMembers(), function(u) {
+    const members = await room.getEncryptionTargetMembers();
+
+    const users = utils.map(members, function(u) {
         return u.userId;
     });
 
-    var self = this;
-    return this._ensureSession(users).then(function() {
-        var payloadFields = {
-            room_id: room.roomId,
-            type: eventType,
-            content: content,
-        };
+    const self = this;
+    await this._ensureSession(users);
 
-        var encryptedContent = {
-            algorithm: olmlib.OLM_ALGORITHM,
-            sender_key: self._olmDevice.deviceCurve25519Key,
-            ciphertext: {},
-        };
+    const payloadFields = {
+        room_id: room.roomId,
+        type: eventType,
+        content: content,
+    };
 
-        for (var i = 0; i < users.length; ++i) {
-            var userId = users[i];
-            var devices = self._crypto.getStoredDevicesForUser(userId);
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: self._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };
 
-            for (var j = 0; j < devices.length; ++j) {
-                var deviceInfo = devices[j];
-                var key = deviceInfo.getIdentityKey();
-                if (key == self._olmDevice.deviceCurve25519Key) {
-                    // don't bother sending to ourself
-                    continue;
-                }
-                if (deviceInfo.verified == DeviceVerification.BLOCKED) {
-                    // don't bother setting up sessions with blocked users
-                    continue;
-                }
+    const promises = [];
 
+    for (let i = 0; i < users.length; ++i) {
+        const userId = users[i];
+        const devices = self._crypto.getStoredDevicesForUser(userId);
+
+        for (let j = 0; j < devices.length; ++j) {
+            const deviceInfo = devices[j];
+            const key = deviceInfo.getIdentityKey();
+            if (key == self._olmDevice.deviceCurve25519Key) {
+                // don't bother sending to ourself
+                continue;
+            }
+            if (deviceInfo.verified == DeviceVerification.BLOCKED) {
+                // don't bother setting up sessions with blocked users
+                continue;
+            }
+
+            promises.push(
                 olmlib.encryptMessageForDevice(
                     encryptedContent.ciphertext,
                     self._userId, self._deviceId, self._olmDevice,
-                    userId, deviceInfo, payloadFields
-                );
-            }
+                    userId, deviceInfo, payloadFields,
+                ),
+            );
         }
+    }
 
-        return encryptedContent;
-    });
+    return await Promise.all(promises).return(encryptedContent);
 };
 
 /**
@@ -153,56 +160,63 @@ utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
  *
  * @param {MatrixEvent} event
  *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting. Rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
  */
-OlmDecryption.prototype.decryptEvent = function(event) {
-    var content = event.getWireContent();
-    var deviceKey = content.sender_key;
-    var ciphertext = content.ciphertext;
+OlmDecryption.prototype.decryptEvent = async function(event) {
+    const content = event.getWireContent();
+    const deviceKey = content.sender_key;
+    const ciphertext = content.ciphertext;
 
     if (!ciphertext) {
-        throw new base.DecryptionError("Missing ciphertext");
+        throw new base.DecryptionError(
+            "OLM_MISSING_CIPHERTEXT",
+            "Missing ciphertext",
+        );
     }
 
     if (!(this._olmDevice.deviceCurve25519Key in ciphertext)) {
-        throw new base.DecryptionError("Not included in recipients");
+        throw new base.DecryptionError(
+            "OLM_NOT_INCLUDED_IN_RECIPIENTS",
+            "Not included in recipients",
+        );
     }
-    var message = ciphertext[this._olmDevice.deviceCurve25519Key];
-    var payloadString;
+    const message = ciphertext[this._olmDevice.deviceCurve25519Key];
+    let payloadString;
 
     try {
-        payloadString = this._decryptMessage(deviceKey, message);
+        payloadString = await this._decryptMessage(deviceKey, message);
     } catch (e) {
-        console.warn(
-            "Failed to decrypt Olm event (id=" +
-                event.getId() + ") from " + deviceKey +
-                ": " + e.message
+        throw new base.DecryptionError(
+            "OLM_BAD_ENCRYPTED_MESSAGE",
+            "Bad Encrypted Message", {
+                sender: deviceKey,
+                err: e,
+            },
         );
-        throw new base.DecryptionError("Bad Encrypted Message");
     }
 
-    var payload = JSON.parse(payloadString);
+    const payload = JSON.parse(payloadString);
 
     // check that we were the intended recipient, to avoid unknown-key attack
     // https://github.com/vector-im/vector-web/issues/2483
     if (payload.recipient != this._userId) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient " +
-            payload.recipient + " does not match our id " + this._userId
-        );
         throw new base.DecryptionError(
-            "Message was intented for " + payload.recipient
+            "OLM_BAD_RECIPIENT",
+            "Message was intented for " + payload.recipient,
         );
     }
 
-    if (payload.recipient_keys.ed25519 !=
-               this._olmDevice.deviceEd25519Key) {
-        console.warn(
-            "Event " + event.getId() + ": Intended recipient ed25519 key " +
-            payload.recipient_keys.ed25519 + " did not match ours"
+    if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
+        throw new base.DecryptionError(
+            "OLM_BAD_RECIPIENT_KEY",
+            "Message not intended for this device", {
+                intended: payload.recipient_keys.ed25519,
+                our_key: this._olmDevice.deviceEd25519Key,
+            },
         );
-        throw new base.DecryptionError("Message not intended for this device");
     }
 
     // check that the original sender matches what the homeserver told us, to
@@ -210,29 +224,32 @@ OlmDecryption.prototype.decryptEvent = function(event) {
     // (this check is also provided via the sender's embedded ed25519 key,
     // which is checked elsewhere).
     if (payload.sender != event.getSender()) {
-        console.warn(
-            "Event " + event.getId() + ": original sender " + payload.sender +
-            " does not match reported sender " + event.getSender()
-        );
         throw new base.DecryptionError(
-            "Message forwarded from " + payload.sender
+            "OLM_FORWARDED_MESSAGE",
+            "Message forwarded from " + payload.sender, {
+                reported_sender: event.getSender(),
+            },
         );
     }
 
     // Olm events intended for a room have a room_id.
     if (payload.room_id !== event.getRoomId()) {
-        console.warn(
-            "Event " + event.getId() + ": original room " + payload.room_id +
-            " does not match reported room " + event.room_id
-        );
         throw new base.DecryptionError(
-            "Message intended for room " + payload.room_id
+            "OLM_BAD_ROOM",
+            "Message intended for room " + payload.room_id, {
+                reported_room: event.room_id,
+            },
         );
     }
 
-    event.setClearData(payload, {curve25519: deviceKey}, payload.keys || {});
-};
+    const claimedKeys = payload.keys || {};
 
+    return {
+        clearEvent: payload,
+        senderCurve25519Key: deviceKey,
+        claimedEd25519Key: claimedKeys.ed25519 || null,
+    };
+};
 
 /**
  * Attempt to decrypt an Olm message
@@ -242,25 +259,29 @@ OlmDecryption.prototype.decryptEvent = function(event) {
  *
  * @return {string} payload, if decrypted successfully.
  */
-OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, message) {
-    var sessionIds = this._olmDevice.getSessionIdsForDevice(theirDeviceIdentityKey);
+OlmDecryption.prototype._decryptMessage = async function(
+    theirDeviceIdentityKey, message,
+) {
+    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
+        theirDeviceIdentityKey,
+    );
 
     // try each session in turn.
-    var decryptionErrors = {};
-    for (var i = 0; i < sessionIds.length; i++) {
-        var sessionId = sessionIds[i];
+    const decryptionErrors = {};
+    for (let i = 0; i < sessionIds.length; i++) {
+        const sessionId = sessionIds[i];
         try {
-            var payload = this._olmDevice.decryptMessage(
-                theirDeviceIdentityKey, sessionId, message.type, message.body
+            const payload = await this._olmDevice.decryptMessage(
+                theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
-            console.log(
+            logger.log(
                 "Decrypted Olm message from " + theirDeviceIdentityKey +
-                    " with session " + sessionId
+                    " with session " + sessionId,
             );
             return payload;
         } catch (e) {
-            var foundSession = this._olmDevice.matchesSession(
-                theirDeviceIdentityKey, sessionId, message.type, message.body
+            const foundSession = await this._olmDevice.matchesSession(
+                theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
 
             if (foundSession) {
@@ -268,7 +289,7 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa
                 // session, so it should have worked.
                 throw new Error(
                     "Error decrypting prekey message with existing session id " +
-                        sessionId + ": " + e.message
+                        sessionId + ": " + e.message,
                 );
             }
 
@@ -288,29 +309,29 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa
 
         throw new Error(
             "Error decrypting non-prekey message with existing sessions: " +
-                JSON.stringify(decryptionErrors)
+                JSON.stringify(decryptionErrors),
         );
     }
 
     // prekey message which doesn't match any existing sessions: make a new
     // session.
 
-    var res;
+    let res;
     try {
-        res = this._olmDevice.createInboundSession(
-            theirDeviceIdentityKey, message.type, message.body
+        res = await this._olmDevice.createInboundSession(
+            theirDeviceIdentityKey, message.type, message.body,
         );
     } catch (e) {
         decryptionErrors["(new)"] = e.message;
         throw new Error(
             "Error decrypting prekey message: " +
-                JSON.stringify(decryptionErrors)
+                JSON.stringify(decryptionErrors),
         );
     }
 
-    console.log(
+    logger.log(
         "created new inbound Olm session ID " +
-            res.session_id + " with " + theirDeviceIdentityKey
+            res.session_id + " with " + theirDeviceIdentityKey,
     );
     return res.payload;
 };

src/crypto/deviceinfo.js

@@ -34,7 +34,11 @@ limitations under the License.
   *      <key type>:<id> -> <base64-encoded key>>
   *
   * @property {module:crypto/deviceinfo.DeviceVerification} verified
-  *     whether the device has been verified by the user
+  *     whether the device has been verified/blocked by the user
+  *
+  * @property {boolean} known
+  *     whether the user knows of this device's existence (useful when warning
+  *     the user that a user has added new devices)
   *
   * @property {Object} unsigned  additional data from the homeserver
   *
@@ -50,6 +54,7 @@ function DeviceInfo(deviceId) {
     this.algorithms = [];
     this.keys = {};
     this.verified = DeviceVerification.UNVERIFIED;
+    this.known = false;
     this.unsigned = {};
 }
 
@@ -62,8 +67,8 @@ function DeviceInfo(deviceId) {
  * @return {module:crypto~DeviceInfo} new DeviceInfo
  */
 DeviceInfo.fromStorage = function(obj, deviceId) {
-    var res = new DeviceInfo(deviceId);
-    for (var prop in obj) {
+    const res = new DeviceInfo(deviceId);
+    for (const prop in obj) {
         if (obj.hasOwnProperty(prop)) {
             res[prop] = obj[prop];
         }
@@ -81,6 +86,7 @@ DeviceInfo.prototype.toStorage = function() {
         algorithms: this.algorithms,
         keys: this.keys,
         verified: this.verified,
+        known: this.known,
         unsigned: this.unsigned,
     };
 };
@@ -130,6 +136,24 @@ DeviceInfo.prototype.isVerified = function() {
     return this.verified == DeviceVerification.VERIFIED;
 };
 
+/**
+ * Returns true if this device is unverified
+ *
+ * @return {Boolean} true if unverified
+ */
+DeviceInfo.prototype.isUnverified = function() {
+    return this.verified == DeviceVerification.UNVERIFIED;
+};
+
+/**
+ * Returns true if the user knows about this device's existence
+ *
+ * @return {Boolean} true if known
+ */
+DeviceInfo.prototype.isKnown = function() {
+    return this.known == true;
+};
+
 /**
  * @enum
  */
@@ -139,7 +163,7 @@ DeviceInfo.DeviceVerification = {
     BLOCKED: -1,
 };
 
-var DeviceVerification = DeviceInfo.DeviceVerification;
+const DeviceVerification = DeviceInfo.DeviceVerification;
 
 /** */
 module.exports = DeviceInfo;

src/crypto/olmlib.js

@@ -20,10 +20,11 @@ limitations under the License.
  * Utilities common to olm encryption algorithms
  */
 
-var q = require('q');
-var anotherjson = require('another-json');
+import Promise from 'bluebird';
+const anotherjson = require('another-json');
 
-var utils = require("../utils");
+const logger = require("../logger");
+const utils = require("../utils");
 
 /**
  * matrix algorithm tag for olm
@@ -48,26 +49,29 @@ module.exports.MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
  * @param {string} recipientUserId
  * @param {module:crypto/deviceinfo} recipientDevice
  * @param {object} payloadFields fields to include in the encrypted payload
+ *
+ * Returns a promise which resolves (to undefined) when the payload
+ *    has been encrypted into `resultsObject`
  */
-module.exports.encryptMessageForDevice = function(
+module.exports.encryptMessageForDevice = async function(
     resultsObject,
     ourUserId, ourDeviceId, olmDevice, recipientUserId, recipientDevice,
-    payloadFields
+    payloadFields,
 ) {
-    var deviceKey = recipientDevice.getIdentityKey();
-    var sessionId = olmDevice.getSessionIdForDevice(deviceKey);
+    const deviceKey = recipientDevice.getIdentityKey();
+    const sessionId = await olmDevice.getSessionIdForDevice(deviceKey);
     if (sessionId === null) {
         // If we don't have a session for a device then
         // we can't encrypt a message for it.
         return;
     }
 
-    console.log(
+    logger.log(
         "Using sessionid " + sessionId + " for device " +
-            recipientUserId + ":" + recipientDevice.deviceId
+            recipientUserId + ":" + recipientDevice.deviceId,
     );
 
-    var payload = {
+    const payload = {
         sender: ourUserId,
         sender_device: ourDeviceId,
 
@@ -99,8 +103,8 @@ module.exports.encryptMessageForDevice = function(
 
     utils.extend(payload, payloadFields);
 
-    resultsObject[deviceKey] = olmDevice.encryptMessage(
-        deviceKey, sessionId, JSON.stringify(payload)
+    resultsObject[deviceKey] = await olmDevice.encryptMessage(
+        deviceKey, sessionId, JSON.stringify(payload),
     );
 };
 
@@ -118,23 +122,25 @@ module.exports.encryptMessageForDevice = function(
  *    an Object mapping from userId to deviceId to
  *    {@link module:crypto~OlmSessionResult}
  */
-module.exports.ensureOlmSessionsForDevices = function(
-    olmDevice, baseApis, devicesByUser
+module.exports.ensureOlmSessionsForDevices = async function(
+    olmDevice, baseApis, devicesByUser,
 ) {
-    var devicesWithoutSession = [
+    const devicesWithoutSession = [
         // [userId, deviceId], ...
     ];
-    var result = {};
+    const result = {};
 
-    for (var userId in devicesByUser) {
-        if (!devicesByUser.hasOwnProperty(userId)) { continue; }
+    for (const userId in devicesByUser) {
+        if (!devicesByUser.hasOwnProperty(userId)) {
+            continue;
+        }
         result[userId] = {};
-        var devices = devicesByUser[userId];
-        for (var j = 0; j < devices.length; j++) {
-            var deviceInfo = devices[j];
-            var deviceId = deviceInfo.deviceId;
-            var key = deviceInfo.getIdentityKey();
-            var sessionId = olmDevice.getSessionIdForDevice(key);
+        const devices = devicesByUser[userId];
+        for (let j = 0; j < devices.length; j++) {
+            const deviceInfo = devices[j];
+            const deviceId = deviceInfo.deviceId;
+            const key = deviceInfo.getIdentityKey();
+            const sessionId = await olmDevice.getSessionIdForDevice(key);
             if (sessionId === null) {
                 devicesWithoutSession.push([userId, deviceId]);
             }
@@ -146,7 +152,7 @@ module.exports.ensureOlmSessionsForDevices = function(
     }
 
     if (devicesWithoutSession.length === 0) {
-        return q(result);
+        return result;
     }
 
     // TODO: this has a race condition - if we try to send another message
@@ -155,78 +161,85 @@ module.exports.ensureOlmSessionsForDevices = function(
     //
     // That should eventually resolve itself, but it's poor form.
 
-    var oneTimeKeyAlgorithm = "signed_curve25519";
-    return baseApis.claimOneTimeKeys(
-        devicesWithoutSession, oneTimeKeyAlgorithm
-    ).then(function(res) {
-        var otk_res = res.one_time_keys || {};
-        for (var userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) { continue; }
-            var userRes = otk_res[userId] || {};
-            var devices = devicesByUser[userId];
-            for (var j = 0; j < devices.length; j++) {
-                var deviceInfo = devices[j];
-                var deviceId = deviceInfo.deviceId;
-                if (result[userId][deviceId].sessionId) {
-                    // we already have a result for this device
-                    continue;
-                }
+    const oneTimeKeyAlgorithm = "signed_curve25519";
+    const res = await baseApis.claimOneTimeKeys(
+        devicesWithoutSession, oneTimeKeyAlgorithm,
+    );
 
-                var deviceRes = userRes[deviceId] || {};
-                var oneTimeKey = null;
-                for (var keyId in deviceRes) {
-                    if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
-                        oneTimeKey = deviceRes[keyId];
-                    }
-                }
-
-                if (!oneTimeKey) {
-                    console.warn(
-                        "No one-time keys (alg=" + oneTimeKeyAlgorithm +
-                            ") for device " + userId + ":" + deviceId
-                    );
-                    continue;
-                }
-
-                var sid = _verifyKeyAndStartSession(
-                    olmDevice, oneTimeKey, userId, deviceInfo
-                );
-                result[userId][deviceId].sessionId = sid;
-            }
+    const otk_res = res.one_time_keys || {};
+    const promises = [];
+    for (const userId in devicesByUser) {
+        if (!devicesByUser.hasOwnProperty(userId)) {
+            continue;
         }
-        return result;
-    });
+        const userRes = otk_res[userId] || {};
+        const devices = devicesByUser[userId];
+        for (let j = 0; j < devices.length; j++) {
+            const deviceInfo = devices[j];
+            const deviceId = deviceInfo.deviceId;
+            if (result[userId][deviceId].sessionId) {
+                // we already have a result for this device
+                continue;
+            }
+
+            const deviceRes = userRes[deviceId] || {};
+            let oneTimeKey = null;
+            for (const keyId in deviceRes) {
+                if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
+                    oneTimeKey = deviceRes[keyId];
+                }
+            }
+
+            if (!oneTimeKey) {
+                logger.warn(
+                    "No one-time keys (alg=" + oneTimeKeyAlgorithm +
+                        ") for device " + userId + ":" + deviceId,
+                );
+                continue;
+            }
+
+            promises.push(
+                _verifyKeyAndStartSession(
+                    olmDevice, oneTimeKey, userId, deviceInfo,
+                ).then((sid) => {
+                    result[userId][deviceId].sessionId = sid;
+                }),
+            );
+        }
+    }
+
+    await Promise.all(promises);
+    return result;
 };
 
-
-function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
-    var deviceId = deviceInfo.deviceId;
+async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
+    const deviceId = deviceInfo.deviceId;
     try {
-        _verifySignature(
+        await _verifySignature(
             olmDevice, oneTimeKey, userId, deviceId,
-            deviceInfo.getFingerprint()
+            deviceInfo.getFingerprint(),
         );
     } catch (e) {
-        console.error(
+        logger.error(
             "Unable to verify signature on one-time key for device " +
-                userId + ":" + deviceId + ":", e
+                userId + ":" + deviceId + ":", e,
         );
         return null;
     }
 
-    var sid;
+    let sid;
     try {
-        sid = olmDevice.createOutboundSession(
-            deviceInfo.getIdentityKey(), oneTimeKey.key
+        sid = await olmDevice.createOutboundSession(
+            deviceInfo.getIdentityKey(), oneTimeKey.key,
         );
     } catch (e) {
         // possibly a bad key
-        console.error("Error starting session with device " +
+        logger.error("Error starting session with device " +
                       userId + ":" + deviceId + ": " + e);
         return null;
     }
 
-    console.log("Started new sessionid " + sid +
+    logger.log("Started new sessionid " + sid +
                 " for device " + userId + ":" + deviceId);
     return sid;
 }
@@ -245,14 +258,17 @@ function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
  * @param {string} signingDeviceId  ID of the device whose signature should be checked
  *
  * @param {string} signingKey   base64-ed ed25519 public key
+ *
+ * Returns a promise which resolves (to undefined) if the the signature is good,
+ * or rejects with an Error if it is bad.
  */
-var _verifySignature = module.exports.verifySignature = function(
-    olmDevice, obj, signingUserId, signingDeviceId, signingKey
+const _verifySignature = module.exports.verifySignature = async function(
+    olmDevice, obj, signingUserId, signingDeviceId, signingKey,
 ) {
-    var signKeyId = "ed25519:" + signingDeviceId;
-    var signatures = obj.signatures || {};
-    var userSigs = signatures[signingUserId] || {};
-    var signature = userSigs[signKeyId];
+    const signKeyId = "ed25519:" + signingDeviceId;
+    const signatures = obj.signatures || {};
+    const userSigs = signatures[signingUserId] || {};
+    const signature = userSigs[signKeyId];
     if (!signature) {
         throw Error("No signature");
     }
@@ -261,9 +277,9 @@ var _verifySignature = module.exports.verifySignature = function(
     // anotherjson
     delete obj.unsigned;
     delete obj.signatures;
-    var json = anotherjson.stringify(obj);
+    const json = anotherjson.stringify(obj);
 
     olmDevice.verifySignature(
-        signingKey, json, signature
+        signingKey, json, signature,
     );
 };

src/crypto/store/base.js

@@ -0,0 +1,34 @@
+/**
+ * Internal module. Defintions for storage for the crypto module
+ *
+ * @module
+ */
+
+/**
+ * Abstraction of things that can store data required for end-to-end encryption
+ *
+ * @interface CryptoStore
+ */
+
+/**
+ * Represents an outgoing room key request
+ *
+ * @typedef {Object} OutgoingRoomKeyRequest
+ *
+ * @property {string} requestId    unique id for this request. Used for both
+ *    an id within the request for later pairing with a cancellation, and for
+ *    the transaction id when sending the to_device messages to our local
+ *    server.
+ *
+ * @property {string?} cancellationTxnId
+ *    transaction id for the cancellation, if any
+ *
+ * @property {Array<{userId: string, deviceId: string}>} recipients
+ *    list of recipients for the request
+ *
+ * @property {module:crypto~RoomKeyRequestBody} requestBody
+ *    parameters for the request.
+ *
+ * @property {Number} state   current state of this request (states are defined
+ *    in {@link module:crypto/OutgoingRoomKeyRequestManager~ROOM_KEY_REQUEST_STATES})
+ */

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -0,0 +1,552 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import utils from '../../utils';
+
+export const VERSION = 6;
+
+/**
+ * Implementation of a CryptoStore which is backed by an existing
+ * IndexedDB connection. Generally you want IndexedDBCryptoStore
+ * which connects to the database and defers to one of these.
+ *
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export class Backend {
+    /**
+     * @param {IDBDatabase} db
+     */
+    constructor(db) {
+        this._db = db;
+
+        // make sure we close the db on `onversionchange` - otherwise
+        // attempts to delete the database will block (and subsequent
+        // attempts to re-create it will also block).
+        db.onversionchange = (ev) => {
+            logger.log(`versionchange for indexeddb ${this._dbName}: closing`);
+            db.close();
+        };
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        const deferred = Promise.defer();
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        txn.onerror = deferred.reject;
+
+        // first see if we already have an entry for this request.
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            if (existing) {
+                // this entry matches the request - return it.
+                logger.log(
+                    `already have key request outstanding for ` +
+                        `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                        `not sending another`,
+                );
+                deferred.resolve(existing);
+                return;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            logger.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                    requestBody.session_id,
+            );
+            txn.oncomplete = () => { deferred.resolve(request); };
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            store.add(request);
+        });
+
+        return deferred.promise;
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        const deferred = Promise.defer();
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        txn.onerror = deferred.reject;
+
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            deferred.resolve(existing);
+        });
+        return deferred.promise;
+    }
+
+    /**
+     * look for an existing room key request in the db
+     *
+     * @private
+     * @param {IDBTransaction} txn  database transaction
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     * @param {Function} callback  function to call with the results of the
+     *    search. Either passed a matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found.
+     */
+    _getOutgoingRoomKeyRequest(txn, requestBody, callback) {
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const idx = store.index("session");
+        const cursorReq = idx.openCursor([
+            requestBody.room_id,
+            requestBody.session_id,
+        ]);
+
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if(!cursor) {
+                // no match found
+                callback(null);
+                return;
+            }
+
+            const existing = cursor.value;
+
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                // got a match
+                callback(existing);
+                return;
+            }
+
+            // look at the next entry in the index
+            cursor.continue();
+        };
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        if (wantedStates.length === 0) {
+            return Promise.resolve(null);
+        }
+
+        // this is a bit tortuous because we need to make sure we do the lookup
+        // in a single transaction, to avoid having a race with the insertion
+        // code.
+
+        // index into the wantedStates array
+        let stateIndex = 0;
+        let result;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (cursor) {
+                // got a match
+                result = cursor.value;
+                return;
+            }
+
+            // try the next state in the list
+            stateIndex++;
+            if (stateIndex >= wantedStates.length) {
+                // no matches
+                return;
+            }
+
+            const wantedState = wantedStates[stateIndex];
+            const cursorReq = ev.target.source.openCursor(wantedState);
+            cursorReq.onsuccess = onsuccess;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const wantedState = wantedStates[stateIndex];
+        const cursorReq = store.index("state").openCursor(wantedState);
+        cursorReq.onsuccess = onsuccess;
+
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        let result = null;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                logger.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${data.state}`,
+                );
+                return;
+            }
+            Object.assign(data, updates);
+            cursor.update(data);
+            result = data;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = onsuccess;
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                logger.warn(
+                    `Cannot delete room key request in state ${data.state} `
+                        + `(expected ${expectedState})`,
+                );
+                return;
+            }
+            cursor.delete();
+        };
+        return promiseifyTxn(txn);
+    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeAccount(txn, newData) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(newData, "-");
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const countReq = objectStore.count();
+        countReq.onsuccess = function() {
+            func(countReq.result);
+        };
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const idx = objectStore.index("deviceKey");
+        const getReq = idx.openCursor(deviceKey);
+        const results = {};
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                results[cursor.value.sessionId] = cursor.value.session;
+                cursor.continue();
+            } else {
+                try {
+                    func(results);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.get([deviceKey, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const objectStore = txn.objectStore("sessions");
+        objectStore.put({deviceKey, sessionId, session});
+    }
+
+    // Inbound group sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.get([senderCurve25519Key, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                try {
+                    func({
+                        senderKey: cursor.value.senderCurve25519Key,
+                        sessionId: cursor.value.sessionId,
+                        sessionData: cursor.value.session,
+                    });
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const addReq = objectStore.add({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+        addReq.onerror = (ev) => {
+            if (addReq.error.name === 'ConstraintError') {
+                // This stops the error from triggering the txn's onerror
+                ev.stopPropagation();
+                // ...and this stops it from aborting the transaction
+                ev.preventDefault();
+                logger.log(
+                    "Ignoring duplicate inbound group session: " +
+                    senderCurve25519Key + " / " + sessionId,
+                );
+            } else {
+                abortWithException(txn, new Error(
+                    "Failed to add inbound group session: " + addReq.error,
+                ));
+            }
+        };
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        objectStore.put({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        const objectStore = txn.objectStore("device_data");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        const objectStore = txn.objectStore("device_data");
+        objectStore.put(deviceData, "-");
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        const objectStore = txn.objectStore("rooms");
+        objectStore.put(roomInfo, roomId);
+    }
+
+    getEndToEndRooms(txn, func) {
+        const rooms = {};
+        const objectStore = txn.objectStore("rooms");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                rooms[cursor.key] = cursor.value;
+                cursor.continue();
+            } else {
+                try {
+                    func(rooms);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    doTxn(mode, stores, func) {
+        const txn = this._db.transaction(stores, mode);
+        const promise = promiseifyTxn(txn);
+        const result = func(txn);
+        return promise.then(() => {
+            return result;
+        });
+    }
+}
+
+export function upgradeDatabase(db, oldVersion) {
+    logger.log(
+        `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
+            + ` to ${VERSION}`,
+    );
+    if (oldVersion < 1) { // The database did not previously exist.
+        createDatabase(db);
+    }
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", {
+            keyPath: ["deviceKey", "sessionId"],
+        });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+    // Expand as needed.
+}
+
+function createDatabase(db) {
+    const outgoingRoomKeyRequestsStore =
+        db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+
+    // we assume that the RoomKeyRequestBody will have room_id and session_id
+    // properties, to make the index efficient.
+    outgoingRoomKeyRequestsStore.createIndex("session",
+        ["requestBody.room_id", "requestBody.session_id"],
+    );
+
+    outgoingRoomKeyRequestsStore.createIndex("state", "state");
+}
+
+/*
+ * Aborts a transaction with a given exception
+ * The transaction promise will be rejected with this exception.
+ */
+function abortWithException(txn, e) {
+    // We cheekily stick our exception onto the transaction object here
+    // We could alternatively make the thing we pass back to the app
+    // an object containing the transaction and exception.
+    txn._mx_abortexception = e;
+    try {
+        txn.abort();
+    } catch (e) {
+        // sometimes we won't be able to abort the transaction
+        // (ie. if it's aborted or completed)
+    }
+}
+
+function promiseifyTxn(txn) {
+    return new Promise((resolve, reject) => {
+        txn.oncomplete = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            resolve();
+        };
+        txn.onerror = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            reject();
+        };
+        txn.onabort = () => reject(txn._mx_abortexception);
+    });
+}

src/crypto/store/indexeddb-crypto-store.js

@@ -0,0 +1,456 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import LocalStorageCryptoStore from './localStorage-crypto-store';
+import MemoryCryptoStore from './memory-crypto-store';
+import * as IndexedDBCryptoStoreBackend from './indexeddb-crypto-store-backend';
+
+/**
+ * Internal module. indexeddb storage for e2e.
+ *
+ * @module
+ */
+
+/**
+ * An implementation of CryptoStore, which is normally backed by an indexeddb,
+ * but with fallback to MemoryCryptoStore.
+ *
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class IndexedDBCryptoStore {
+    /**
+     * Create a new IndexedDBCryptoStore
+     *
+     * @param {IDBFactory} indexedDB  global indexedDB instance
+     * @param {string} dbName   name of db to connect to
+     */
+    constructor(indexedDB, dbName) {
+        this._indexedDB = indexedDB;
+        this._dbName = dbName;
+        this._backendPromise = null;
+    }
+
+    /**
+     * Ensure the database exists and is up-to-date, or fall back to
+     * a local storage or in-memory store.
+     *
+     * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
+     * or a MemoryCryptoStore
+     */
+    _connect() {
+        if (this._backendPromise) {
+            return this._backendPromise;
+        }
+
+        this._backendPromise = new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            logger.log(`connecting to indexeddb ${this._dbName}`);
+
+            const req = this._indexedDB.open(
+                this._dbName, IndexedDBCryptoStoreBackend.VERSION,
+            );
+
+            req.onupgradeneeded = (ev) => {
+                const db = ev.target.result;
+                const oldVersion = ev.oldVersion;
+                IndexedDBCryptoStoreBackend.upgradeDatabase(db, oldVersion);
+            };
+
+            req.onblocked = () => {
+                logger.log(
+                    `can't yet open IndexedDBCryptoStore because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                reject(ev.target.error);
+            };
+
+            req.onsuccess = (r) => {
+                const db = r.target.result;
+
+                logger.log(`connected to indexeddb ${this._dbName}`);
+                resolve(new IndexedDBCryptoStoreBackend.Backend(db));
+            };
+        }).then((backend) => {
+            // Edge has IndexedDB but doesn't support compund keys which we use fairly extensively.
+            // Try a dummy query which will fail if the browser doesn't support compund keys, so
+            // we can fall back to a different backend.
+            return backend.doTxn(
+                'readonly',
+                [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                (txn) => {
+                    backend.getEndToEndInboundGroupSession('', '', txn, () => {});
+                }).then(() => {
+                    return backend;
+                },
+            );
+        }).catch((e) => {
+            logger.warn(
+                `unable to connect to indexeddb ${this._dbName}` +
+                    `: falling back to localStorage store: ${e}`,
+            );
+            return new LocalStorageCryptoStore(global.localStorage);
+        }).catch((e) => {
+            logger.warn(
+                `unable to open localStorage: falling back to in-memory store: ${e}`,
+            );
+            return new MemoryCryptoStore();
+        });
+
+        return this._backendPromise;
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            logger.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this._indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                logger.log(
+                    `can't yet delete IndexedDBCryptoStore because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                reject(ev.target.error);
+            };
+
+            req.onsuccess = () => {
+                logger.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        }).catch((e) => {
+            // in firefox, with indexedDB disabled, this fails with a
+            // DOMError. We treat this as non-fatal, so that people can
+            // still use the app.
+            logger.warn(`unable to delete IndexedDBCryptoStore: ${e}`);
+        });
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        return this._connect().then((backend) => {
+            return backend.getOrAddOutgoingRoomKeyRequest(request);
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequest(requestBody);
+        });
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequestByState(wantedStates);
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        return this._connect().then((backend) => {
+            return backend.updateOutgoingRoomKeyRequest(
+                requestId, expectedState, updates,
+            );
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        return this._connect().then((backend) => {
+            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
+        });
+    }
+
+    // Olm Account
+
+    /*
+     * Get the account pickle from the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the account pickle
+     */
+    getAccount(txn, func) {
+        this._backendPromise.value().getAccount(txn, func);
+    }
+
+    /*
+     * Write the account pickle to the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} newData The new account pickle to store.
+     */
+    storeAccount(txn, newData) {
+        this._backendPromise.value().storeAccount(txn, newData);
+    }
+
+    // Olm sessions
+
+    /**
+     * Returns the number of end-to-end sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(int)} func Called with the count of sessions
+     */
+    countEndToEndSessions(txn, func) {
+        this._backendPromise.value().countEndToEndSessions(txn, func);
+    }
+
+    /**
+     * Retrieve a specific end-to-end session between the logged-in user
+     * and another device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID of the session to retrieve
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndSession(deviceKey, sessionId, txn, func);
+    }
+
+    /**
+     * Retrieve the end-to-end sessions between the logged-in user and another
+     * device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSessions(deviceKey, txn, func) {
+        this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func);
+    }
+
+    /**
+     * Store a session between the logged-in user and another device
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID for this end-to-end session.
+     * @param {string} session Base64 encoded end-to-end session.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        this._backendPromise.value().storeEndToEndSession(
+            deviceKey, sessionId, session, txn,
+        );
+    }
+
+    // Inbound group saessions
+
+    /**
+     * Retrieve the end-to-end inbound group session for a given
+     * server key and session ID
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
+    }
+
+    /**
+     * Fetches all inbound group sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called once for each group session
+     *     in the store with an object having keys {senderKey, sessionId,
+     *     sessionData}, then once with null to indicate the end of the list.
+     */
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndInboundGroupSessions(txn, func);
+    }
+
+    /**
+     * Adds an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, the session will not be added.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    /**
+     * Writes an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, it will be overwritten.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    // End-to-end device tracking
+
+    /**
+     * Store the state of all tracked devices
+     * This contains devices for each user, a tracking state for each user
+     * and a sync token matching the point in time the snapshot represents.
+     * These all need to be written out in full each time such that the snapshot
+     * is always consistent, so they are stored in one object.
+     *
+     * @param {Object} deviceData
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._backendPromise.value().storeEndToEndDeviceData(deviceData, txn);
+    }
+
+    /**
+     * Get the state of all tracked devices
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the
+     *     device data
+     */
+    getEndToEndDeviceData(txn, func) {
+        this._backendPromise.value().getEndToEndDeviceData(txn, func);
+    }
+
+    // End to End Rooms
+
+    /**
+     * Store the end-to-end state for a room.
+     * @param {string} roomId The room's ID.
+     * @param {object} roomInfo The end-to-end info for the room.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._backendPromise.value().storeEndToEndRoom(roomId, roomInfo, txn);
+    }
+
+    /**
+     * Get an object of roomId->roomInfo for all e2e rooms in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the end to end encrypted rooms
+     */
+    getEndToEndRooms(txn, func) {
+        this._backendPromise.value().getEndToEndRooms(txn, func);
+    }
+
+    /**
+     * Perform a transaction on the crypto store. Any store methods
+     * that require a transaction (txn) object to be passed in may
+     * only be called within a callback of either this function or
+     * one of the store functions operating on the same transaction.
+     *
+     * @param {string} mode 'readwrite' if you need to call setter
+     *     functions with this transaction. Otherwise, 'readonly'.
+     * @param {string[]} stores List IndexedDBCryptoStore.STORE_*
+     *     options representing all types of object that will be
+     *     accessed or written to with this transaction.
+     * @param {function(*)} func Function called with the
+     *     transaction object: an opaque object that should be passed
+     *     to store functions.
+     * @return {Promise} Promise that resolves with the result of the `func`
+     *     when the transaction is complete. If the backend is
+     *     async (ie. the indexeddb backend) any of the callback
+     *     functions throwing an exception will cause this promise to
+     *     reject with that exception. On synchronous backends, the
+     *     exception will propagate to the caller of the getFoo method.
+     */
+    doTxn(mode, stores, func) {
+        return this._connect().then((backend) => {
+            return backend.doTxn(mode, stores, func);
+        });
+    }
+}
+
+IndexedDBCryptoStore.STORE_ACCOUNT = 'account';
+IndexedDBCryptoStore.STORE_SESSIONS = 'sessions';
+IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS = 'inbound_group_sessions';
+IndexedDBCryptoStore.STORE_DEVICE_DATA = 'device_data';
+IndexedDBCryptoStore.STORE_ROOMS = 'rooms';

src/crypto/store/localStorage-crypto-store.js

@@ -0,0 +1,212 @@
+/*
+Copyright 2017, 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import MemoryCryptoStore from './memory-crypto-store.js';
+
+/**
+ * Internal module. Partial localStorage backed storage for e2e.
+ * This is not a full crypto store, just the in-memory store with
+ * some things backed by localStorage. It exists because indexedDB
+ * is broken in Firefox private mode or set to, "will not remember
+ * history".
+ *
+ * @module
+ */
+
+const E2E_PREFIX = "crypto.";
+const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
+const KEY_DEVICE_DATA = E2E_PREFIX + "device_data";
+const KEY_INBOUND_SESSION_PREFIX = E2E_PREFIX + "inboundgroupsessions/";
+const KEY_ROOMS_PREFIX = E2E_PREFIX + "rooms/";
+
+function keyEndToEndSessions(deviceKey) {
+    return E2E_PREFIX + "sessions/" + deviceKey;
+}
+
+function keyEndToEndInboundGroupSession(senderKey, sessionId) {
+    return KEY_INBOUND_SESSION_PREFIX + senderKey + "/" + sessionId;
+}
+
+function keyEndToEndRoomsPrefix(roomId) {
+    return KEY_ROOMS_PREFIX + roomId;
+}
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class LocalStorageCryptoStore extends MemoryCryptoStore {
+    constructor(webStore) {
+        super();
+        this.store = webStore;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        let count = 0;
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) ++count;
+        }
+        func(count);
+    }
+
+    _getEndToEndSessions(deviceKey, txn, func) {
+        return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const sessions = this._getEndToEndSessions(deviceKey);
+        func(sessions[sessionId] || {});
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._getEndToEndSessions(deviceKey) || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const sessions = this._getEndToEndSessions(deviceKey) || {};
+        sessions[sessionId] = session;
+        setJsonItem(
+            this.store, keyEndToEndSessions(deviceKey), sessions,
+        );
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        ));
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                // we can't use split, as the components we are trying to split out
+                // might themselves contain '/' characters. We rely on the
+                // senderKey being a (32-byte) curve25519 key, base64-encoded
+                // (hence 43 characters long).
+
+                func({
+                    senderKey: key.substr(KEY_INBOUND_SESSION_PREFIX.length, 43),
+                    sessionId: key.substr(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    sessionData: getJsonItem(this.store, key),
+                });
+            }
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const existing = getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        );
+        if (!existing) {
+            this.storeEndToEndInboundGroupSession(
+                senderCurve25519Key, sessionId, sessionData, txn,
+            );
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        setJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+            sessionData,
+        );
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        func(getJsonItem(
+            this.store, KEY_DEVICE_DATA,
+        ));
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        setJsonItem(
+            this.store, KEY_DEVICE_DATA, deviceData,
+        );
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        setJsonItem(
+            this.store, keyEndToEndRoomsPrefix(roomId), roomInfo,
+        );
+    }
+
+    getEndToEndRooms(txn, func) {
+        const result = {};
+        const prefix = keyEndToEndRoomsPrefix('');
+
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(prefix)) {
+                const roomId = key.substr(prefix.length);
+                result[roomId] = getJsonItem(this.store, key);
+            }
+        }
+        func(result);
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
+        return Promise.resolve();
+    }
+
+    // Olm account
+
+    getAccount(txn, func) {
+        const account = getJsonItem(this.store, KEY_END_TO_END_ACCOUNT);
+        func(account);
+    }
+
+    storeAccount(txn, newData) {
+        setJsonItem(
+            this.store, KEY_END_TO_END_ACCOUNT, newData,
+        );
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
+}
+
+function getJsonItem(store, key) {
+    try {
+        // if the key is absent, store.getItem() returns null, and
+        // JSON.parse(null) === null, so this returns null.
+        return JSON.parse(store.getItem(key));
+    } catch (e) {
+        logger.log("Error: Failed to get key %s: %s", key, e.stack || e);
+        logger.log(e.stack);
+    }
+    return null;
+}
+
+function setJsonItem(store, key, val) {
+    store.setItem(key, JSON.stringify(val));
+}

src/crypto/store/memory-crypto-store.js

@@ -0,0 +1,302 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import utils from '../../utils';
+
+/**
+ * Internal module. in-memory storage for e2e.
+ *
+ * @module
+ */
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class MemoryCryptoStore {
+    constructor() {
+        this._outgoingRoomKeyRequests = [];
+        this._account = null;
+
+        // Map of {devicekey -> {sessionId -> session pickle}}
+        this._sessions = {};
+        // Map of {senderCurve25519Key+'/'+sessionId -> session data object}
+        this._inboundGroupSessions = {};
+        // Opaque device data object
+        this._deviceData = null;
+        // roomId -> Opaque roomInfo object
+        this._rooms = {};
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        return Promise.resolve();
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        return Promise.try(() => {
+            // first see if we already have an entry for this request.
+            const existing = this._getOutgoingRoomKeyRequest(requestBody);
+
+            if (existing) {
+                // this entry matches the request - return it.
+                logger.log(
+                    `already have key request outstanding for ` +
+                    `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                    `not sending another`,
+                );
+                return existing;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            logger.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                requestBody.session_id,
+            );
+            this._outgoingRoomKeyRequests.push(request);
+            return request;
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return Promise.resolve(this._getOutgoingRoomKeyRequest(requestBody));
+    }
+
+    /**
+     * Looks for existing room key request, and returns the result synchronously.
+     *
+     * @internal
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {module:crypto/store/base~OutgoingRoomKeyRequest?}
+     *    the matching request, or null if not found
+     */
+    _getOutgoingRoomKeyRequest(requestBody) {
+        for (const existing of this._outgoingRoomKeyRequests) {
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                return existing;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            for (const state of wantedStates) {
+                if (req.state === state) {
+                    return Promise.resolve(req);
+                }
+            }
+        }
+        return Promise.resolve(null);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                logger.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${req.state}`,
+                );
+                return Promise.resolve(null);
+            }
+            Object.assign(req, updates);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        for (let i = 0; i < this._outgoingRoomKeyRequests.length; i++) {
+            const req = this._outgoingRoomKeyRequests[i];
+
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                logger.warn(
+                    `Cannot delete room key request in state ${req.state} `
+                    + `(expected ${expectedState})`,
+                );
+                return Promise.resolve(null);
+            }
+
+            this._outgoingRoomKeyRequests.splice(i, 1);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        func(this._account);
+    }
+
+    storeAccount(txn, newData) {
+        this._account = newData;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        return Object.keys(this._sessions).length;
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const deviceSessions = this._sessions[deviceKey] || {};
+        func(deviceSessions[sessionId] || null);
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._sessions[deviceKey] || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        let deviceSessions = this._sessions[deviceKey];
+        if (deviceSessions === undefined) {
+            deviceSessions = {};
+            this._sessions[deviceKey] = deviceSessions;
+        }
+        deviceSessions[sessionId] = session;
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] || null);
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (const key of Object.keys(this._inboundGroupSessions)) {
+            // we can't use split, as the components we are trying to split out
+            // might themselves contain '/' characters. We rely on the
+            // senderKey being a (32-byte) curve25519 key, base64-encoded
+            // (hence 43 characters long).
+
+            func({
+                senderKey: key.substr(0, 43),
+                sessionId: key.substr(44),
+                sessionData: this._inboundGroupSessions[key],
+            });
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const k = senderCurve25519Key+'/'+sessionId;
+        if (this._inboundGroupSessions[k] === undefined) {
+            this._inboundGroupSessions[k] = sessionData;
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] = sessionData;
+    }
+
+    // Device Data
+
+    getEndToEndDeviceData(txn, func) {
+        func(this._deviceData);
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._deviceData = deviceData;
+    }
+
+    // E2E rooms
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._rooms[roomId] = roomInfo;
+    }
+
+    getEndToEndRooms(txn, func) {
+        func(this._rooms);
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
+}

src/errors.js

@@ -0,0 +1,25 @@
+// can't just do InvalidStoreError extends Error
+// because of http://babeljs.io/docs/usage/caveats/#classes
+function InvalidStoreError(reason, value) {
+    const message = `Store is invalid because ${reason}, ` +
+        `please stop the client, delete all data and start the client again`;
+    const instance = Reflect.construct(Error, [message]);
+    Reflect.setPrototypeOf(instance, Reflect.getPrototypeOf(this));
+    instance.reason = reason;
+    instance.value = value;
+    return instance;
+}
+
+InvalidStoreError.TOGGLED_LAZY_LOADING = "TOGGLED_LAZY_LOADING";
+
+InvalidStoreError.prototype = Object.create(Error.prototype, {
+  constructor: {
+    value: Error,
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  },
+});
+Reflect.setPrototypeOf(InvalidStoreError, Error);
+
+module.exports.InvalidStoreError = InvalidStoreError;

src/filter-component.js

@@ -27,10 +27,9 @@ limitations under the License.
  */
 function _matches_wildcard(actual_value, filter_value) {
     if (filter_value.endsWith("*")) {
-        var type_prefix = filter_value.slice(0, -1);
+        const type_prefix = filter_value.slice(0, -1);
         return actual_value.substr(0, type_prefix.length) === type_prefix;
-    }
-    else {
+    } else {
         return actual_value === filter_value;
     }
 }
@@ -71,7 +70,7 @@ FilterComponent.prototype.check = function(event) {
         event.getRoomId(),
         event.getSender(),
         event.getType(),
-        event.getContent() ? event.getContent().url !== undefined : false
+        event.getContent() ? event.getContent().url !== undefined : false,
     );
 };
 
@@ -84,32 +83,38 @@ FilterComponent.prototype.check = function(event) {
  * @return {bool} true if the event fields match the filter
  */
 FilterComponent.prototype._checkFields =
-    function(room_id, sender, event_type, contains_url)
-{
-    var literal_keys = {
-        "rooms": function(v) { return room_id === v; },
-        "senders": function(v) { return sender === v; },
-        "types": function(v) { return _matches_wildcard(event_type, v); },
+    function(room_id, sender, event_type, contains_url) {
+    const literal_keys = {
+        "rooms": function(v) {
+            return room_id === v;
+        },
+        "senders": function(v) {
+            return sender === v;
+        },
+        "types": function(v) {
+            return _matches_wildcard(event_type, v);
+        },
     };
 
-    var self = this;
-    Object.keys(literal_keys).forEach(function(name) {
-        var match_func = literal_keys[name];
-        var not_name = "not_" + name;
-        var disallowed_values = self[not_name];
-        if (disallowed_values.map(match_func)) {
+    const self = this;
+    for (let n=0; n < Object.keys(literal_keys).length; n++) {
+        const name = Object.keys(literal_keys)[n];
+        const match_func = literal_keys[name];
+        const not_name = "not_" + name;
+        const disallowed_values = self[not_name];
+        if (disallowed_values.filter(match_func).length > 0) {
             return false;
         }
 
-        var allowed_values = self[name];
+        const allowed_values = self[name];
         if (allowed_values) {
             if (!allowed_values.map(match_func)) {
                 return false;
             }
         }
-    });
+    }
 
-    var contains_url_filter = this.filter_json.contains_url;
+    const contains_url_filter = this.filter_json.contains_url;
     if (contains_url_filter !== undefined) {
         if (contains_url_filter !== contains_url) {
             return false;

src/filter.js

@@ -18,7 +18,7 @@ limitations under the License.
  * @module filter
  */
 
-var FilterComponent = require("./filter-component");
+const FilterComponent = require("./filter-component");
 
 /**
  * @param {Object} obj
@@ -26,9 +26,9 @@ var FilterComponent = require("./filter-component");
  * @param {*} val
  */
 function setProp(obj, keyNesting, val) {
-    var nestedKeys = keyNesting.split(".");
-    var currentObj = obj;
-    for (var i = 0; i < (nestedKeys.length - 1); i++) {
+    const nestedKeys = keyNesting.split(".");
+    let currentObj = obj;
+    for (let i = 0; i < (nestedKeys.length - 1); i++) {
         if (!currentObj[nestedKeys[i]]) {
             currentObj[nestedKeys[i]] = {};
         }
@@ -51,6 +51,17 @@ function Filter(userId, filterId) {
     this.definition = {};
 }
 
+Filter.LAZY_LOADING_MESSAGES_FILTER = {
+    lazy_load_members: true,
+};
+
+Filter.LAZY_LOADING_SYNC_FILTER = {
+    room: {
+        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
+    },
+};
+
+
 /**
  * Get the ID of this filter on your homeserver (if known)
  * @return {?Number} The filter ID
@@ -106,10 +117,10 @@ Filter.prototype.setDefinition = function(definition) {
     //   "event_fields": ["type", "content", "sender"]
     // }
 
-    var room_filter_json = definition.room;
+    const room_filter_json = definition.room;
 
     // consider the top level rooms/not_rooms filter
-    var room_filter_fields = {};
+    const room_filter_fields = {};
     if (room_filter_json) {
         if (room_filter_json.rooms) {
             room_filter_fields.rooms = room_filter_json.rooms;
@@ -123,7 +134,7 @@ Filter.prototype.setDefinition = function(definition) {
 
     this._room_filter = new FilterComponent(room_filter_fields);
     this._room_timeline_filter = new FilterComponent(
-        room_filter_json ? (room_filter_json.timeline || {}) : {}
+        room_filter_json ? (room_filter_json.timeline || {}) : {},
     );
 
     // don't bother porting this from synapse yet:
@@ -183,7 +194,7 @@ Filter.prototype.setIncludeLeaveRooms = function(includeLeave) {
  * @return {Filter}
  */
 Filter.fromJson = function(userId, filterId, jsonObj) {
-    var filter = new Filter(userId, filterId);
+    const filter = new Filter(userId, filterId);
     filter.setDefinition(jsonObj);
     return filter;
 };

src/http-api.js

@@ -18,13 +18,15 @@ limitations under the License.
  * This is an internal module. See {@link MatrixHttpApi} for the public class.
  * @module http-api
  */
-var q = require("q");
-var utils = require("./utils");
+import Promise from 'bluebird';
+const parseContentType = require('content-type').parse;
+
+const utils = require("./utils");
 
 // we use our own implementation of setTimeout, so that if we get suspended in
 // the middle of a /sync, we cancel the sync as soon as we awake, rather than
 // waiting for the delay to elapse.
-var callbacks = require("./realtime-callbacks");
+const callbacks = require("./realtime-callbacks");
 
 /*
 TODO:
@@ -64,7 +66,7 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
  * @param {string} opts.prefix Required. The matrix client prefix to use, e.g.
  * '/_matrix/client/r0'. See PREFIX_R0 and PREFIX_UNSTABLE for constants.
  *
- * @param {bool=} opts.onlyData True to return only the 'data' component of the
+ * @param {boolean} opts.onlyData True to return only the 'data' component of the
  * response (e.g. the parsed HTTP body). If false, requests will return an
  * object with the properties <tt>code</tt>, <tt>headers</tt> and <tt>data</tt>.
  *
@@ -74,12 +76,15 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
  * requests.
  * @param {Number=} opts.localTimeoutMs The default maximum amount of time to wait
  * before timing out the request. If not specified, there is no timeout.
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
  */
 module.exports.MatrixHttpApi = function MatrixHttpApi(event_emitter, opts) {
     utils.checkObjectHasKeys(opts, ["baseUrl", "request", "prefix"]);
     opts.onlyData = opts.onlyData || false;
     this.event_emitter = event_emitter;
     this.opts = opts;
+    this.useAuthorizationHeader = Boolean(opts.useAuthorizationHeader);
     this.uploads = [];
 };
 
@@ -91,13 +96,13 @@ module.exports.MatrixHttpApi.prototype = {
      *          path and query parameters respectively.
      */
     getContentUri: function() {
-        var params = {
-            access_token: this.opts.accessToken
+        const params = {
+            access_token: this.opts.accessToken,
         };
         return {
             base: this.opts.baseUrl,
             path: "/_matrix/media/v1/upload",
-            params: params
+            params: params,
         };
     },
 
@@ -113,6 +118,10 @@ module.exports.MatrixHttpApi.prototype = {
      * @param {string=} opts.name   Name to give the file on the server. Defaults
      *   to <tt>file.name</tt>.
      *
+     * @param {boolean=} opts.includeFilename if false will not send the filename,
+     *   e.g for encrypted file uploads where filename leaks are undesirable.
+     *   Defaults to true.
+     *
      * @param {string=} opts.type   Content-type for the upload. Defaults to
      *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
      *
@@ -129,6 +138,10 @@ module.exports.MatrixHttpApi.prototype = {
      *    invoke on success/failure. See the promise return values for more
      *    information.
      *
+     * @param {Function=} opts.progressHandler Optional. Called when a chunk of
+     *    data has been uploaded, with an object containing the fields `loaded`
+     *    (number of bytes transferred) and `total` (total size, if known).
+     *
      * @return {module:client.Promise} Resolves to response object, as
      *    determined by this.opts.onlyData, opts.rawResponse, and
      *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
@@ -143,18 +156,21 @@ module.exports.MatrixHttpApi.prototype = {
             opts = {};
         }
 
+        // default opts.includeFilename to true (ignoring falsey values)
+        const includeFilename = opts.includeFilename !== false;
+
         // if the file doesn't have a mime type, use a default since
         // the HS errors if we don't supply one.
-        var contentType = opts.type || file.type || 'application/octet-stream';
-        var fileName = opts.name || file.name;
+        const contentType = opts.type || file.type || 'application/octet-stream';
+        const fileName = opts.name || file.name;
 
         // we used to recommend setting file.stream to the thing to upload on
         // nodejs.
-        var body = file.stream ? file.stream : file;
+        const body = file.stream ? file.stream : file;
 
         // backwards-compatibility hacks where we used to do different things
         // between browser and node.
-        var rawResponse = opts.rawResponse;
+        let rawResponse = opts.rawResponse;
         if (rawResponse === undefined) {
             if (global.XMLHttpRequest) {
                 rawResponse = false;
@@ -163,20 +179,20 @@ module.exports.MatrixHttpApi.prototype = {
                     "Returning the raw JSON from uploadContent(). Future " +
                     "versions of the js-sdk will change this default, to " +
                     "return the parsed object. Set opts.rawResponse=false " +
-                    "to change this behaviour now."
+                    "to change this behaviour now.",
                 );
                 rawResponse = true;
             }
         }
 
-        var onlyContentUri = opts.onlyContentUri;
+        let onlyContentUri = opts.onlyContentUri;
         if (!rawResponse && onlyContentUri === undefined) {
             if (global.XMLHttpRequest) {
                 console.warn(
                     "Returning only the content-uri from uploadContent(). " +
                     "Future versions of the js-sdk will change this " +
                     "default, to return the whole response object. Set " +
-                    "opts.onlyContentUri=false to change this behaviour now."
+                    "opts.onlyContentUri=false to change this behaviour now.",
                 );
                 onlyContentUri = true;
             } else {
@@ -192,17 +208,17 @@ module.exports.MatrixHttpApi.prototype = {
         // (browser-request doesn't support progress either, which is also kind
         // of important here)
 
-        var upload = { loaded: 0, total: 0 };
-        var promise;
+        const upload = { loaded: 0, total: 0 };
+        let promise;
 
         // XMLHttpRequest doesn't parse JSON for us. request normally does, but
         // we're setting opts.json=false so that it doesn't JSON-encode the
         // request, which also means it doesn't JSON-decode the response. Either
         // way, we have to JSON-parse the response ourselves.
-        var bodyParser = null;
+        let bodyParser = null;
         if (!rawResponse) {
             bodyParser = function(rawBody) {
-                var body = JSON.parse(rawBody);
+                let body = JSON.parse(rawBody);
                 if (onlyContentUri) {
                     body = body.content_uri;
                     if (body === undefined) {
@@ -214,12 +230,12 @@ module.exports.MatrixHttpApi.prototype = {
         }
 
         if (global.XMLHttpRequest) {
-            var defer = q.defer();
-            var xhr = new global.XMLHttpRequest();
+            const defer = Promise.defer();
+            const xhr = new global.XMLHttpRequest();
             upload.xhr = xhr;
-            var cb = requestCallback(defer, opts.callback, this.opts.onlyData);
+            const cb = requestCallback(defer, opts.callback, this.opts.onlyData);
 
-            var timeout_fn = function() {
+            const timeout_fn = function() {
                 xhr.abort();
                 cb(new Error('Timeout'));
             };
@@ -255,13 +271,34 @@ module.exports.MatrixHttpApi.prototype = {
                 upload.loaded = ev.loaded;
                 upload.total = ev.total;
                 xhr.timeout_timer = callbacks.setTimeout(timeout_fn, 30000);
-                defer.notify(ev);
+                if (opts.progressHandler) {
+                    opts.progressHandler({
+                        loaded: ev.loaded,
+                        total: ev.total,
+                    });
+                }
             });
-            var url = this.opts.baseUrl + "/_matrix/media/v1/upload";
-            url += "?access_token=" + encodeURIComponent(this.opts.accessToken);
-            url += "&filename=" + encodeURIComponent(fileName);
+            let url = this.opts.baseUrl + "/_matrix/media/v1/upload";
+
+            const queryArgs = [];
+
+            if (includeFilename && fileName) {
+                queryArgs.push("filename=" + encodeURIComponent(fileName));
+            }
+
+            if (!this.useAuthorizationHeader) {
+                queryArgs.push("access_token="
+                    + encodeURIComponent(this.opts.accessToken));
+            }
+
+            if (queryArgs.length > 0) {
+                url += "?" + queryArgs.join("&");
+            }
 
             xhr.open("POST", url);
+            if (this.useAuthorizationHeader) {
+                xhr.setRequestHeader("Authorization", "Bearer " + this.opts.accessToken);
+            }
             xhr.setRequestHeader("Content-Type", contentType);
             xhr.send(body);
             promise = defer.promise;
@@ -269,9 +306,11 @@ module.exports.MatrixHttpApi.prototype = {
             // dirty hack (as per _request) to allow the upload to be cancelled.
             promise.abort = xhr.abort.bind(xhr);
         } else {
-            var queryParams = {
-                filename: fileName,
-            };
+            const queryParams = {};
+
+            if (includeFilename && fileName) {
+                queryParams.filename = fileName;
+            }
 
             promise = this.authedRequest(
                 opts.callback, "POST", "/upload", queryParams, body, {
@@ -279,15 +318,15 @@ module.exports.MatrixHttpApi.prototype = {
                     headers: {"Content-Type": contentType},
                     json: false,
                     bodyParser: bodyParser,
-                }
+                },
             );
         }
 
-        var self = this;
+        const self = this;
 
         // remove the upload from the list on completion
-        var promise0 = promise.finally(function() {
-            for (var i = 0; i < self.uploads.length; ++i) {
+        const promise0 = promise.finally(function() {
+            for (let i = 0; i < self.uploads.length; ++i) {
                 if (self.uploads[i] === upload) {
                     self.uploads.splice(i, 1);
                     return;
@@ -317,20 +356,20 @@ module.exports.MatrixHttpApi.prototype = {
     },
 
     idServerRequest: function(callback, method, path, params, prefix) {
-        var fullUri = this.opts.idBaseUrl + prefix + path;
+        const fullUri = this.opts.idBaseUrl + prefix + path;
 
         if (callback !== undefined && !utils.isFunction(callback)) {
             throw Error(
-                "Expected callback to be a function but got " + typeof callback
+                "Expected callback to be a function but got " + typeof callback,
             );
         }
 
-        var opts = {
+        const opts = {
             uri: fullUri,
             method: method,
             withCredentials: false,
             json: false,
-            _matrix_opts: this.opts
+            _matrix_opts: this.opts,
         };
         if (method == 'GET') {
             opts.qs = params;
@@ -338,10 +377,10 @@ module.exports.MatrixHttpApi.prototype = {
             opts.form = params;
         }
 
-        var defer = q.defer();
+        const defer = Promise.defer();
         this.opts.request(
             opts,
-            requestCallback(defer, callback, this.opts.onlyData)
+            requestCallback(defer, callback, this.opts.onlyData),
         );
         // ID server does not always take JSON, so we can't use requests' 'json'
         // option as we do with the home server, but it does return JSON, so
@@ -364,7 +403,8 @@ module.exports.MatrixHttpApi.prototype = {
      *
      * @param {Object} data The HTTP JSON body.
      *
-     * @param {Object=} opts additional options
+     * @param {Object|Number=} opts additional options. If a number is specified,
+     * this is treated as `opts.localTimeoutMs`.
      *
      * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
      * timing out the request. If not specified, there is no timeout.
@@ -385,24 +425,51 @@ module.exports.MatrixHttpApi.prototype = {
         if (!queryParams) {
             queryParams = {};
         }
-        if (!queryParams.access_token) {
-            queryParams.access_token = this.opts.accessToken;
+        if (this.useAuthorizationHeader) {
+            if (isFinite(opts)) {
+                // opts used to be localTimeoutMs
+                opts = {
+                    localTimeoutMs: opts,
+                };
+            }
+            if (!opts) {
+                opts = {};
+            }
+            if (!opts.headers) {
+                opts.headers = {};
+            }
+            if (!opts.headers.Authorization) {
+                opts.headers.Authorization = "Bearer " + this.opts.accessToken;
+            }
+            if (queryParams.access_token) {
+                delete queryParams.access_token;
+            }
+        } else {
+            if (!queryParams.access_token) {
+                queryParams.access_token = this.opts.accessToken;
+            }
         }
 
-        var request_promise = this.request(
-            callback, method, path, queryParams, data, opts
+        const requestPromise = this.request(
+            callback, method, path, queryParams, data, opts,
         );
 
-        var self = this;
-        request_promise.catch(function(err) {
+        const self = this;
+        requestPromise.catch(function(err) {
             if (err.errcode == 'M_UNKNOWN_TOKEN') {
                 self.event_emitter.emit("Session.logged_out");
+            } else if (err.errcode == 'M_CONSENT_NOT_GIVEN') {
+                self.event_emitter.emit(
+                    "no_consent",
+                    err.message,
+                    err.data.consent_uri,
+                );
             }
         });
 
         // return the original promise, otherwise tests break due to it having to
         // go around the event loop one more time to process the result of the request
-        return request_promise;
+        return requestPromise;
     },
 
     /**
@@ -437,11 +504,11 @@ module.exports.MatrixHttpApi.prototype = {
      */
     request: function(callback, method, path, queryParams, data, opts) {
         opts = opts || {};
-        var prefix = opts.prefix !== undefined ? opts.prefix : this.opts.prefix;
-        var fullUri = this.opts.baseUrl + prefix + path;
+        const prefix = opts.prefix !== undefined ? opts.prefix : this.opts.prefix;
+        const fullUri = this.opts.baseUrl + prefix + path;
 
         return this.requestOtherUrl(
-            callback, method, fullUri, queryParams, data, opts
+            callback, method, fullUri, queryParams, data, opts,
         );
     },
 
@@ -476,7 +543,7 @@ module.exports.MatrixHttpApi.prototype = {
             callback, method, path, queryParams, data, {
                 localTimeoutMs: localTimeoutMs,
                 prefix: prefix,
-            }
+            },
         );
     },
 
@@ -511,7 +578,7 @@ module.exports.MatrixHttpApi.prototype = {
             callback, method, path, queryParams, data, {
                 localTimeoutMs: localTimeoutMs,
                 prefix: prefix,
-            }
+            },
         );
     },
 
@@ -551,12 +618,12 @@ module.exports.MatrixHttpApi.prototype = {
         } else if (isFinite(opts)) {
             // opts used to be localTimeoutMs
             opts = {
-                localTimeoutMs: opts
+                localTimeoutMs: opts,
             };
         }
 
         return this._request(
-            callback, method, uri, queryParams, data, opts
+            callback, method, uri, queryParams, data, opts,
         );
     },
 
@@ -572,7 +639,7 @@ module.exports.MatrixHttpApi.prototype = {
      * @return {string} URL
      */
     getUrl: function(path, queryParams, prefix) {
-        var queryString = "";
+        let queryString = "";
         if (queryParams) {
             queryString = "?" + utils.encodeParams(queryParams);
         }
@@ -608,42 +675,75 @@ module.exports.MatrixHttpApi.prototype = {
     _request: function(callback, method, uri, queryParams, data, opts) {
         if (callback !== undefined && !utils.isFunction(callback)) {
             throw Error(
-                "Expected callback to be a function but got " + typeof callback
+                "Expected callback to be a function but got " + typeof callback,
             );
         }
         opts = opts || {};
 
-        var self = this;
+        const self = this;
         if (this.opts.extraParams) {
-            for (var key in this.opts.extraParams) {
-                if (!this.opts.extraParams.hasOwnProperty(key)) { continue; }
+            for (const key in this.opts.extraParams) {
+                if (!this.opts.extraParams.hasOwnProperty(key)) {
+                    continue;
+                }
                 queryParams[key] = this.opts.extraParams[key];
             }
         }
 
-        var json = opts.json === undefined ? true : opts.json;
+        const headers = utils.extend({}, opts.headers || {});
+        const json = opts.json === undefined ? true : opts.json;
+        let bodyParser = opts.bodyParser;
 
-        var defer = q.defer();
+        // we handle the json encoding/decoding here, because request and
+        // browser-request make a mess of it. Specifically, they attempt to
+        // json-decode plain-text error responses, which in turn means that the
+        // actual error gets swallowed by a SyntaxError.
 
-        var timeoutId;
-        var timedOut = false;
-        var req;
-        var localTimeoutMs = opts.localTimeoutMs || this.opts.localTimeoutMs;
-        if (localTimeoutMs) {
-            timeoutId = callbacks.setTimeout(function() {
-                timedOut = true;
-                if (req && req.abort) {
-                    req.abort();
-                }
-                defer.reject(new module.exports.MatrixError({
-                    error: "Locally timed out waiting for a response",
-                    errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
-                    timeout: localTimeoutMs
-                }));
-            }, localTimeoutMs);
+        if (json) {
+            if (data) {
+                data = JSON.stringify(data);
+                headers['content-type'] = 'application/json';
+            }
+
+            if (!headers['accept']) {
+                headers['accept'] = 'application/json';
+            }
+
+            if (bodyParser === undefined) {
+                bodyParser = function(rawBody) {
+                    return JSON.parse(rawBody);
+                };
+            }
         }
 
-        var reqPromise = defer.promise;
+        const defer = Promise.defer();
+
+        let timeoutId;
+        let timedOut = false;
+        let req;
+        const localTimeoutMs = opts.localTimeoutMs || this.opts.localTimeoutMs;
+
+        const resetTimeout = () => {
+            if (localTimeoutMs) {
+                if (timeoutId) {
+                    callbacks.clearTimeout(timeoutId);
+                }
+                timeoutId = callbacks.setTimeout(function() {
+                    timedOut = true;
+                    if (req && req.abort) {
+                        req.abort();
+                    }
+                    defer.reject(new module.exports.MatrixError({
+                        error: "Locally timed out waiting for a response",
+                        errcode: "ORG.MATRIX.JSSDK_TIMEOUT",
+                        timeout: localTimeoutMs,
+                    }));
+                }, localTimeoutMs);
+            }
+        };
+        resetTimeout();
+
+        const reqPromise = defer.promise;
 
         try {
             req = this.opts.request(
@@ -652,11 +752,13 @@ module.exports.MatrixHttpApi.prototype = {
                     method: method,
                     withCredentials: false,
                     qs: queryParams,
+                    qsStringifyOptions: opts.qsStringifyOptions,
+                    useQuerystring: true,
                     body: data,
-                    json: json,
+                    json: false,
                     timeout: localTimeoutMs,
-                    headers: opts.headers || {},
-                    _matrix_opts: this.opts
+                    headers: headers || {},
+                    _matrix_opts: this.opts,
                 },
                 function(err, response, body) {
                     if (localTimeoutMs) {
@@ -666,31 +768,38 @@ module.exports.MatrixHttpApi.prototype = {
                         }
                     }
 
-                    // if json is falsy, we won't parse any error response, so need
-                    // to do so before turning it into a MatrixError
-                    var parseErrorJson = !json;
-                    var handlerFn = requestCallback(
+                    const handlerFn = requestCallback(
                         defer, callback, self.opts.onlyData,
-                        parseErrorJson,
-                        opts.bodyParser
+                        bodyParser,
                     );
                     handlerFn(err, response, body);
-                }
+                },
             );
-            if (req && req.abort) {
+            if (req) {
+                // This will only work in a browser, where opts.request is the
+                // `browser-request` import. Currently `request` does not support progress
+                // updates - see https://github.com/request/request/pull/2346.
+                // `browser-request` returns an XHRHttpRequest which exposes `onprogress`
+                if ('onprogress' in req) {
+                    req.onprogress = (e) => {
+                        // Prevent the timeout from rejecting the deferred promise if progress is
+                        // seen with the request
+                        resetTimeout();
+                    };
+                }
+
                 // FIXME: This is EVIL, but I can't think of a better way to expose
                 // abort() operations on underlying HTTP requests :(
-                reqPromise.abort = req.abort.bind(req);
+                if (req.abort) reqPromise.abort = req.abort.bind(req);
             }
-        }
-        catch (ex) {
+        } catch (ex) {
             defer.reject(ex);
             if (callback) {
                 callback(ex);
             }
         }
         return reqPromise;
-    }
+    },
 };
 
 /*
@@ -698,15 +807,18 @@ module.exports.MatrixHttpApi.prototype = {
  * that will either resolve or reject the given defer as well as invoke the
  * given userDefinedCallback (if any).
  *
- * If onlyData is true, the defer/callback is invoked with the body of the
- * response, otherwise the result code.
+ * HTTP errors are transformed into javascript errors and the deferred is rejected.
  *
- * If parseErrorJson is true, we will JSON.parse the body if we get a 4xx error.
+ * If bodyParser is given, it is used to transform the body of the successful
+ * responses before passing to the defer/callback.
+ *
+ * If onlyData is true, the defer/callback is invoked with the body of the
+ * response, otherwise the result object (with `code` and `data` fields)
  *
  */
-var requestCallback = function(
+const requestCallback = function(
     defer, userDefinedCallback, onlyData,
-    parseErrorJson, bodyParser
+    bodyParser,
 ) {
     userDefinedCallback = userDefinedCallback || function() {};
 
@@ -714,31 +826,26 @@ var requestCallback = function(
         if (!err) {
             try {
                 if (response.statusCode >= 400) {
-                    if (parseErrorJson) {
-                        // we won't have json-decoded the response.
-                        body = JSON.parse(body);
-                    }
-                    err = new module.exports.MatrixError(body);
+                    err = parseErrorResponse(response, body);
                 } else if (bodyParser) {
                     body = bodyParser(body);
                 }
             } catch (e) {
-                err = e;
-            }
-            if (err) {
-                err.httpStatus = response.statusCode;
+                err = new Error(`Error parsing server response: ${e}`);
             }
         }
 
         if (err) {
             defer.reject(err);
             userDefinedCallback(err);
-        }
-        else {
-            var res = {
+        } else {
+            const res = {
                 code: response.statusCode,
+
+                // XXX: why do we bother with this? it doesn't work for
+                // XMLHttpRequest, so clearly we don't use it.
                 headers: response.headers,
-                data: body
+                data: body,
             };
             defer.resolve(onlyData ? body : res);
             userDefinedCallback(null, onlyData ? body : res);
@@ -746,6 +853,67 @@ var requestCallback = function(
     };
 };
 
+/**
+ * Attempt to turn an HTTP error response into a Javascript Error.
+ *
+ * If it is a JSON response, we will parse it into a MatrixError. Otherwise
+ * we return a generic Error.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @param {String} body raw body of the response
+ * @returns {Error}
+ */
+function parseErrorResponse(response, body) {
+    const httpStatus = response.statusCode;
+    const contentType = getResponseContentType(response);
+
+    let err;
+    if (contentType) {
+        if (contentType.type === 'application/json') {
+            err = new module.exports.MatrixError(JSON.parse(body));
+        } else if (contentType.type === 'text/plain') {
+            err = new Error(`Server returned ${httpStatus} error: ${body}`);
+        }
+    }
+
+    if (!err) {
+        err = new Error(`Server returned ${httpStatus} error`);
+    }
+    err.httpStatus = httpStatus;
+    return err;
+}
+
+
+/**
+ * extract the Content-Type header from the response object, and
+ * parse it to a `{type, parameters}` object.
+ *
+ * returns null if no content-type header could be found.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @returns {{type: String, parameters: Object}?} parsed content-type header, or null if not found
+ */
+function getResponseContentType(response) {
+    let contentType;
+    if (response.getResponseHeader) {
+        // XMLHttpRequest provides getResponseHeader
+        contentType = response.getResponseHeader("Content-Type");
+    } else if (response.headers) {
+        // request provides http.IncomingMessage which has a message.headers map
+        contentType = response.headers['content-type'] || null;
+    }
+
+    if (!contentType) {
+        return null;
+    }
+
+    try {
+        return parseContentType(contentType);
+    } catch(e) {
+        throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
+    }
+}
+
 /**
  * Construct a Matrix error. This is a JavaScript Error with additional
  * information specific to the standard Matrix error response.

src/indexeddb-worker.js

@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Separate exports file for the indexeddb web worker, which is designed
+ * to be used separately
+ */
+
+/** The {@link module:indexeddb-store-worker~IndexedDBStoreWorker} class. */
+module.exports.IndexedDBStoreWorker = require("./store/indexeddb-store-worker.js");
+

src/interactive-auth.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,9 +17,13 @@ limitations under the License.
 "use strict";
 
 /** @module interactive-auth */
-var q = require("q");
+import Promise from 'bluebird';
+const url = require("url");
 
-var utils = require("./utils");
+const utils = require("./utils");
+
+const EMAIL_STAGE_TYPE = "m.login.email.identity";
+const MSISDN_STAGE_TYPE = "m.login.msisdn";
 
 /**
  * Abstracts the logic used to drive the interactive auth process.
@@ -37,44 +42,123 @@ var utils = require("./utils");
  *
  * @param {object} opts  options object
  *
+ * @param {object} opts.matrixClient A matrix client to use for the auth process
+ *
  * @param {object?} opts.authData error response from the last request. If
  *    null, a request will be made with no auth before starting.
  *
- * @param {function(object?): module:client.Promise} opts.doRequest
- *     called with the new auth dict to submit the request. Should return a
+ * @param {function(object?, bool?): module:client.Promise} opts.doRequest
+ *     called with the new auth dict to submit the request and a flag set
+ *     to true if this request is a background request. Should return a
  *     promise which resolves to the successful response or rejects with a
  *     MatrixError.
  *
- * @param {function(string, object?)} opts.startAuthStage
- *     called to ask the UI to start a particular auth stage. The arguments
- *     are: the login type (eg m.login.password); and (if the last request
- *     returned an error), an error object, with fields 'errcode' and 'error'.
+ * @param {function(string, object?)} opts.stateUpdated
+ *     called when the status of the UI auth changes, ie. when the state of
+ *     an auth stage changes of when the auth flow moves to a new stage.
+ *     The arguments are: the login type (eg m.login.password); and an object
+ *     which is either an error or an informational object specific to the
+ *     login type. If the 'errcode' key is defined, the object is an error,
+ *     and has keys:
+ *         errcode: string, the textual error code, eg. M_UNKNOWN
+ *         error: string, human readable string describing the error
+ *
+ *     The login type specific objects are as follows:
+ *         m.login.email.identity:
+ *          * emailSid: string, the sid of the active email auth session
+ *
+ * @param {object?} opts.inputs Inputs provided by the user and used by different
+ *     stages of the auto process. The inputs provided will affect what flow is chosen.
+ *
+ * @param {string?} opts.inputs.emailAddress An email address. If supplied, a flow
+ *     using email verification will be chosen.
+ *
+ * @param {string?} opts.inputs.phoneCountry An ISO two letter country code. Gives
+ *     the country that opts.phoneNumber should be resolved relative to.
+ *
+ * @param {string?} opts.inputs.phoneNumber A phone number. If supplied, a flow
+ *     using phone number validation will be chosen.
+ *
+ * @param {string?} opts.sessionId If resuming an existing interactive auth session,
+ *     the sessionId of that session.
+ *
+ * @param {string?} opts.clientSecret If resuming an existing interactive auth session,
+ *     the client secret for that session
+ *
+ * @param {string?} opts.emailSid If returning from having completed m.login.email.identity
+ *     auth, the sid for the email verification session.
  *
  */
 function InteractiveAuth(opts) {
-    this._data = opts.authData;
+    this._matrixClient = opts.matrixClient;
+    this._data = opts.authData || {};
     this._requestCallback = opts.doRequest;
-    this._startAuthStageCallback = opts.startAuthStage;
+    // startAuthStage included for backwards compat
+    this._stateUpdatedCallback = opts.stateUpdated || opts.startAuthStage;
     this._completionDeferred = null;
+    this._inputs = opts.inputs || {};
+
+    if (opts.sessionId) this._data.session = opts.sessionId;
+    this._clientSecret = opts.clientSecret || this._matrixClient.generateClientSecret();
+    this._emailSid = opts.emailSid;
+    if (this._emailSid === undefined) this._emailSid = null;
+
+    this._currentStage = null;
 }
 
 InteractiveAuth.prototype = {
     /**
      * begin the authentication process.
      *
-     * @return {module:client.Promise}  which resolves to the response on success,
-     * or rejects with the error on failure.
+     * @return {module:client.Promise} which resolves to the response on success,
+     * or rejects with the error on failure. Rejects with NoAuthFlowFoundError if
+     *     no suitable authentication flow can be found
      */
     attemptAuth: function() {
-        this._completionDeferred = q.defer();
+        this._completionDeferred = Promise.defer();
 
-        if (!this._data) {
-            this._doRequest(null);
-        } else {
-            this._startNextAuthStage();
+        // wrap in a promise so that if _startNextAuthStage
+        // throws, it rejects the promise in a consistent way
+        return Promise.resolve().then(() => {
+            // if we have no flows, try a request (we'll have
+            // just a session ID in _data if resuming)
+            if (!this._data.flows) {
+                this._doRequest(this._data);
+            } else {
+                this._startNextAuthStage();
+            }
+            return this._completionDeferred.promise;
+        });
+    },
+
+    /**
+     * Poll to check if the auth session or current stage has been
+     * completed out-of-band. If so, the attemptAuth promise will
+     * be resolved.
+     */
+    poll: function() {
+        if (!this._data.session) return;
+
+        let authDict = {};
+        if (this._currentStage == EMAIL_STAGE_TYPE) {
+            // The email can be validated out-of-band, but we need to provide the
+            // creds so the HS can go & check it.
+            if (this._emailSid) {
+                const idServerParsedUrl = url.parse(
+                    this._matrixClient.getIdentityServerUrl(),
+                );
+                authDict = {
+                    type: EMAIL_STAGE_TYPE,
+                    threepid_creds: {
+                        sid: this._emailSid,
+                        client_secret: this._clientSecret,
+                        id_server: idServerParsedUrl.host,
+                    },
+                };
+            }
         }
 
-        return this._completionDeferred.promise;
+        this.submitAuthDict(authDict, true);
     },
 
     /**
@@ -86,6 +170,16 @@ InteractiveAuth.prototype = {
         return this._data ? this._data.session : undefined;
     },
 
+    /**
+     * get the client secret used for validation sessions
+     * with the ID server.
+     *
+     * @return {string} client secret
+     */
+    getClientSecret: function() {
+        return this._clientSecret;
+    },
+
     /**
      * get the server params for a given stage
      *
@@ -93,7 +187,7 @@ InteractiveAuth.prototype = {
      * @return {object?} any parameters from the server for this stage
      */
     getStageParams: function(loginType) {
-        var params = {};
+        let params = {};
         if (this._data && this._data.params) {
             params = this._data.params;
         }
@@ -108,19 +202,44 @@ InteractiveAuth.prototype = {
      * @param {object} authData new auth dict to send to the server. Should
      *    include a `type` propterty denoting the login type, as well as any
      *    other params for that stage.
+     * @param {bool} background If true, this request failing will not result
+     *    in the attemptAuth promise being rejected. This can be set to true
+     *    for requests that just poll to see if auth has been completed elsewhere.
      */
-    submitAuthDict: function(authData) {
+    submitAuthDict: function(authData, background) {
         if (!this._completionDeferred) {
             throw new Error("submitAuthDict() called before attemptAuth()");
         }
 
         // use the sessionid from the last request.
-        var auth = {
+        const auth = {
             session: this._data.session,
         };
         utils.extend(auth, authData);
 
-        this._doRequest(auth);
+        this._doRequest(auth, background);
+    },
+
+    /**
+     * Gets the sid for the email validation session
+     * Specific to m.login.email.identity
+     *
+     * @returns {string} The sid of the email auth session
+     */
+    getEmailSid: function() {
+        return this._emailSid;
+    },
+
+    /**
+     * Sets the sid for the email validation session
+     * This must be set in order to successfully poll for completion
+     * of the email validation.
+     * Specific to m.login.email.identity
+     *
+     * @param {string} sid The sid for the email validation session
+     */
+    setEmailSid: function(sid) {
+        this._emailSid = sid;
     },
 
     /**
@@ -129,54 +248,98 @@ InteractiveAuth.prototype = {
      *
      * @private
      * @param {object?} auth new auth dict, including session id
+     * @param {bool?} background If true, this request is a background poll, so it
+     *    failing will not result in the attemptAuth promise being rejected.
+     *    This can be set to true for requests that just poll to see if auth has
+     *    been completed elsewhere.
      */
-    _doRequest: function(auth) {
-        var self = this;
+    _doRequest: function(auth, background) {
+        const self = this;
 
         // hackery to make sure that synchronous exceptions end up in the catch
         // handler (without the additional event loop entailed by q.fcall or an
-        // extra q().then)
-        var prom;
+        // extra Promise.resolve().then)
+        let prom;
         try {
-            prom = this._requestCallback(auth);
+            prom = this._requestCallback(auth, background);
         } catch (e) {
-            prom = q.reject(e);
+            prom = Promise.reject(e);
         }
 
-        prom.then(
+        prom = prom.then(
             function(result) {
                 console.log("result from request: ", result);
                 self._completionDeferred.resolve(result);
             }, function(error) {
-                if (error.httpStatus !== 401 || !error.data || !error.data.flows) {
+                // sometimes UI auth errors don't come with flows
+                const errorFlows = error.data ? error.data.flows : null;
+                const haveFlows = Boolean(self._data.flows) || Boolean(errorFlows);
+                if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                     // doesn't look like an interactive-auth failure. fail the whole lot.
                     throw error;
                 }
+                // if the error didn't come with flows, completed flows or session ID,
+                // copy over the ones we have. Synapse sometimes sends responses without
+                // any UI auth data (eg. when polling for email validation, if the email
+                // has not yet been validated). This appears to be a Synapse bug, which
+                // we workaround here.
+                if (!error.data.flows && !error.data.completed && !error.data.session) {
+                    error.data.flows = self._data.flows;
+                    error.data.completed = self._data.completed;
+                    error.data.session = self._data.session;
+                }
                 self._data = error.data;
                 self._startNextAuthStage();
-            }
-        ).catch(this._completionDeferred.reject).done();
+            },
+        );
+        if (!background) {
+            prom = prom.catch((e) => {
+                this._completionDeferred.reject(e);
+            });
+        } else {
+            // We ignore all failures here (even non-UI auth related ones)
+            // since we don't want to suddenly fail if the internet connection
+            // had a blip whilst we were polling
+            prom = prom.catch((error) => {
+                console.log("Ignoring error from UI auth: " + error);
+            });
+        }
+        prom.done();
     },
 
     /**
      * Pick the next stage and call the callback
      *
      * @private
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _startNextAuthStage: function() {
-        var nextStage = this._chooseStage();
+        const nextStage = this._chooseStage();
         if (!nextStage) {
             throw new Error("No incomplete flows from the server");
         }
+        this._currentStage = nextStage;
+
+        if (nextStage == 'm.login.dummy') {
+            this.submitAuthDict({
+                type: 'm.login.dummy',
+            });
+            return;
+        }
 
-        var stageError = null;
         if (this._data.errcode || this._data.error) {
-            stageError = {
+            this._stateUpdatedCallback(nextStage, {
                 errcode: this._data.errcode || "",
                 error: this._data.error || "",
-            };
+            });
+            return;
         }
-        this._startAuthStageCallback(nextStage, stageError);
+
+        const stageStatus = {};
+        if (nextStage == EMAIL_STAGE_TYPE) {
+            stageStatus.emailSid = this._emailSid;
+        }
+        this._stateUpdatedCallback(nextStage, stageStatus);
     },
 
     /**
@@ -184,25 +347,65 @@ InteractiveAuth.prototype = {
      *
      * @private
      * @return {string?} login type
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _chooseStage: function() {
-        var flow = this._chooseFlow();
+        const flow = this._chooseFlow();
         console.log("Active flow => %s", JSON.stringify(flow));
-        var nextStage = this._firstUncompletedStage(flow);
+        const nextStage = this._firstUncompletedStage(flow);
         console.log("Next stage: %s", nextStage);
         return nextStage;
     },
 
     /**
      * Pick one of the flows from the returned list
+     * If a flow using all of the inputs is found, it will
+     * be returned, otherwise, null will be returned.
+     *
+     * Only flows using all given inputs are chosen because it
+     * is likley to be surprising if the user provides a
+     * credential and it is not used. For example, for registration,
+     * this could result in the email not being used which would leave
+     * the account with no means to reset a password.
      *
      * @private
      * @return {object} flow
+     * @throws {NoAuthFlowFoundError} If no suitable authentication flow can be found
      */
     _chooseFlow: function() {
-        var flows = this._data.flows || [];
-        // always use the first flow for now
-        return flows[0];
+        const flows = this._data.flows || [];
+
+        // we've been given an email or we've already done an email part
+        const haveEmail = Boolean(this._inputs.emailAddress) || Boolean(this._emailSid);
+        const haveMsisdn = (
+            Boolean(this._inputs.phoneCountry) &&
+            Boolean(this._inputs.phoneNumber)
+        );
+
+        for (const flow of flows) {
+            let flowHasEmail = false;
+            let flowHasMsisdn = false;
+            for (const stage of flow.stages) {
+                if (stage === EMAIL_STAGE_TYPE) {
+                    flowHasEmail = true;
+                } else if (stage == MSISDN_STAGE_TYPE) {
+                    flowHasMsisdn = true;
+                }
+            }
+
+            if (flowHasEmail == haveEmail && flowHasMsisdn == haveMsisdn) {
+                return flow;
+            }
+        }
+        // Throw an error with a fairly generic description, but with more
+        // information such that the app can give a better one if so desired.
+        const err = new Error("No appropriate authentication flow found");
+        err.name = 'NoAuthFlowFoundError';
+        err.required_stages = [];
+        if (haveEmail) err.required_stages.push(EMAIL_STAGE_TYPE);
+        if (haveMsisdn) err.required_stages.push(MSISDN_STAGE_TYPE);
+        err.available_flows = flows;
+        throw err;
     },
 
     /**
@@ -213,9 +416,9 @@ InteractiveAuth.prototype = {
      * @return {string} login type
      */
     _firstUncompletedStage: function(flow) {
-        var completed = (this._data || {}).completed || [];
-        for (var i = 0; i < flow.stages.length; ++i) {
-            var stageType = flow.stages[i];
+        const completed = (this._data || {}).completed || [];
+        for (let i = 0; i < flow.stages.length; ++i) {
+            const stageType = flow.stages[i];
             if (completed.indexOf(stageType) === -1) {
                 return stageType;
             }

src/logger.js

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 André Jaenisch
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module logger
+ */
+const log = require("loglevel");
+
+// This is to demonstrate, that you can use any namespace you want.
+// Namespaces allow you to turn on/off the logging for specific parts of the
+// application.
+// An idea would be to control this via an environment variable (on Node.js).
+// See https://www.npmjs.com/package/debug to see how this could be implemented
+// Part of #332 is introducing a logging library in the first place.
+const DEFAULT_NAME_SPACE = "matrix";
+const logger = log.getLogger(DEFAULT_NAME_SPACE);
+logger.setLevel(log.levels.DEBUG);
+
+/**
+ * Drop-in replacement for <code>console</code> using {@link https://www.npmjs.com/package/loglevel|loglevel}.
+ * Can be tailored down to specific use cases if needed.
+*/
+module.exports = logger;

src/matrix.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,23 +16,32 @@ limitations under the License.
 */
 "use strict";
 
+/** The {@link module:ContentHelpers} object */
+module.exports.ContentHelpers = require("./content-helpers");
 /** The {@link module:models/event.MatrixEvent|MatrixEvent} class. */
 module.exports.MatrixEvent = require("./models/event").MatrixEvent;
 /** The {@link module:models/event.EventStatus|EventStatus} enum. */
 module.exports.EventStatus = require("./models/event").EventStatus;
 /** The {@link module:store/memory.MatrixInMemoryStore|MatrixInMemoryStore} class. */
 module.exports.MatrixInMemoryStore = require("./store/memory").MatrixInMemoryStore;
-/** The {@link module:store/webstorage~WebStorageStore|WebStorageStore} class.
- * <strong>Work in progress; unstable.</strong> */
-module.exports.WebStorageStore = require("./store/webstorage");
+/** The {@link module:store/indexeddb.IndexedDBStore|IndexedDBStore} class. */
+module.exports.IndexedDBStore = require("./store/indexeddb").IndexedDBStore;
+/** The {@link module:store/indexeddb.IndexedDBStoreBackend|IndexedDBStoreBackend} class. */
+module.exports.IndexedDBStoreBackend = require("./store/indexeddb").IndexedDBStoreBackend;
+/** The {@link module:sync-accumulator.SyncAccumulator|SyncAccumulator} class. */
+module.exports.SyncAccumulator = require("./sync-accumulator");
 /** The {@link module:http-api.MatrixHttpApi|MatrixHttpApi} class. */
 module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi;
 /** The {@link module:http-api.MatrixError|MatrixError} class. */
 module.exports.MatrixError = require("./http-api").MatrixError;
+/** The {@link module:errors.InvalidStoreError|InvalidStoreError} class. */
+module.exports.InvalidStoreError = require("./errors").InvalidStoreError;
 /** The {@link module:client.MatrixClient|MatrixClient} class. */
 module.exports.MatrixClient = require("./client").MatrixClient;
 /** The {@link module:models/room|Room} class. */
 module.exports.Room = require("./models/room");
+/** The {@link module:models/group|Group} class. */
+module.exports.Group = require("./models/group");
 /** The {@link module:models/event-timeline~EventTimeline} class. */
 module.exports.EventTimeline = require("./models/event-timeline");
 /** The {@link module:models/event-timeline-set~EventTimelineSet} class. */
@@ -59,6 +69,11 @@ module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
 module.exports.InteractiveAuth = require("./interactive-auth");
 
 
+module.exports.MemoryCryptoStore =
+    require("./crypto/store/memory-crypto-store").default;
+module.exports.IndexedDBCryptoStore =
+    require("./crypto/store/indexeddb-crypto-store").default;
+
 /**
  * Create a new Matrix Call.
  * @function
@@ -69,9 +84,33 @@ module.exports.InteractiveAuth = require("./interactive-auth");
  */
 module.exports.createNewMatrixCall = require("./webrtc/call").createNewMatrixCall;
 
+
+/**
+ * Set an audio output device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallAudioOutput = require('./webrtc/call').setAudioOutput;
+/**
+ * Set an audio input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallAudioInput = require('./webrtc/call').setAudioInput;
+/**
+ * Set a video input device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallVideoInput = require('./webrtc/call').setVideoInput;
+
+
 // expose the underlying request object so different environments can use
 // different request libs (e.g. request or browser-request)
-var request;
+let request;
 /**
  * The function used to perform HTTP requests. Only use this if you want to
  * use a different HTTP library, e.g. Angular's <code>$http</code>. This should
@@ -97,12 +136,25 @@ module.exports.getRequest = function() {
  * @param {requestWrapperFunction} wrapper The wrapping function.
  */
 module.exports.wrapRequest = function(wrapper) {
-    var origRequest = request;
+    const origRequest = request;
     request = function(options, callback) {
         return wrapper(origRequest, options, callback);
     };
 };
 
+
+let cryptoStoreFactory = () => new module.exports.MemoryCryptoStore;
+
+/**
+ * Configure a different factory to be used for creating crypto stores
+ *
+ * @param {Function} fac  a function which will return a new
+ *    {@link module:crypto.store.base~CryptoStore}.
+ */
+module.exports.setCryptoStoreFactory = function(fac) {
+    cryptoStoreFactory = fac;
+};
+
 /**
  * Construct a Matrix Client. Similar to {@link module:client~MatrixClient}
  * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -115,6 +167,13 @@ module.exports.wrapRequest = function(wrapper) {
  * {@link module:scheduler~MatrixScheduler}.
  * @param {requestFunction} opts.request If not set, defaults to the function
  * supplied to {@link request} which defaults to the request module from NPM.
+ *
+ * @param {module:crypto.store.base~CryptoStore=} opts.cryptoStore
+ *    crypto store implementation. Calls the factory supplied to
+ *    {@link setCryptoStoreFactory} if unspecified; or if no factory has been
+ *    specified, uses a default implementation (indexeddb in the browser,
+ *    in-memory otherwise).
+ *
  * @return {MatrixClient} A new matrix client.
  * @see {@link module:client~MatrixClient} for the full list of options for
  * <code>opts</code>.
@@ -122,14 +181,15 @@ module.exports.wrapRequest = function(wrapper) {
 module.exports.createClient = function(opts) {
     if (typeof opts === "string") {
         opts = {
-            "baseUrl": opts
+            "baseUrl": opts,
         };
     }
     opts.request = opts.request || request;
     opts.store = opts.store || new module.exports.MatrixInMemoryStore({
-      localStorage: global.localStorage
+      localStorage: global.localStorage,
     });
     opts.scheduler = opts.scheduler || new module.exports.MatrixScheduler();
+    opts.cryptoStore = opts.cryptoStore || cryptoStoreFactory();
     return new module.exports.MatrixClient(opts);
 };
 

src/models/event-timeline-set.js

@@ -17,14 +17,14 @@ limitations under the License.
 /**
  * @module models/event-timeline-set
  */
-var EventEmitter = require("events").EventEmitter;
-var utils = require("../utils");
-var EventTimeline = require("./event-timeline");
+const EventEmitter = require("events").EventEmitter;
+const utils = require("../utils");
+const EventTimeline = require("./event-timeline");
 
 // var DEBUG = false;
-var DEBUG = true;
+const DEBUG = true;
 
-var debuglog;
+let debuglog;
 if (DEBUG) {
     // using bind means that we get to keep useful line numbers in the console
     debuglog = console.log.bind(console);
@@ -106,8 +106,7 @@ EventTimelineSet.prototype.getPendingEvents = function() {
 
     if (this._filter) {
         return this._filter.filterRoomTimeline(this.room.getPendingEvents());
-    }
-    else {
+    } else {
         return this.room.getPendingEvents();
     }
 };
@@ -137,7 +136,7 @@ EventTimelineSet.prototype.eventIdToTimeline = function(eventId) {
  * @param {String} newEventId  event ID of the replacement event
  */
 EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
-    var existingTimeline = this._eventIdToTimeline[oldEventId];
+    const existingTimeline = this._eventIdToTimeline[oldEventId];
     if (existingTimeline) {
         delete this._eventIdToTimeline[oldEventId];
         this._eventIdToTimeline[newEventId] = existingTimeline;
@@ -150,41 +149,53 @@ EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
  * <p>This is used when /sync returns a 'limited' timeline.
  *
  * @param {string=} backPaginationToken   token for back-paginating the new timeline
- * @param {?bool} flush  Whether to flush the non-live timelines too.
+ * @param {string=} forwardPaginationToken token for forward-paginating the old live timeline,
+ * if absent or null, all timelines are reset.
  *
  * @fires module:client~MatrixClient#event:"Room.timelineReset"
  */
-EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
-    var newTimeline;
+EventTimelineSet.prototype.resetLiveTimeline = function(
+    backPaginationToken, forwardPaginationToken,
+) {
+    // Each EventTimeline has RoomState objects tracking the state at the start
+    // and end of that timeline. The copies at the end of the live timeline are
+    // special because they will have listeners attached to monitor changes to
+    // the current room state, so we move this RoomState from the end of the
+    // current live timeline to the end of the new one and, if necessary,
+    // replace it with a newly created one. We also make a copy for the start
+    // of the new timeline.
 
-    if (!this._timelineSupport || flush) {
-        // if timeline support is disabled, forget about the old timelines
-        newTimeline = new EventTimeline(this);
+    // if timeline support is disabled, forget about the old timelines
+    const resetAllTimelines = !this._timelineSupport || !forwardPaginationToken;
+
+    const oldTimeline = this._liveTimeline;
+    const newTimeline = resetAllTimelines ?
+        oldTimeline.forkLive(EventTimeline.FORWARDS) :
+        oldTimeline.fork(EventTimeline.FORWARDS);
+
+    if (resetAllTimelines) {
         this._timelines = [newTimeline];
         this._eventIdToTimeline = {};
     } else {
-        newTimeline = this.addTimeline();
+        this._timelines.push(newTimeline);
     }
 
-    // initialise the state in the new timeline from our last known state
-    var evMap = this._liveTimeline.getState(EventTimeline.FORWARDS).events;
-    var events = [];
-    for (var evtype in evMap) {
-        if (!evMap.hasOwnProperty(evtype)) { continue; }
-        for (var stateKey in evMap[evtype]) {
-            if (!evMap[evtype].hasOwnProperty(stateKey)) { continue; }
-            events.push(evMap[evtype][stateKey]);
-        }
+    if (forwardPaginationToken) {
+        // Now set the forward pagination token on the old live timeline
+        // so it can be forward-paginated.
+        oldTimeline.setPaginationToken(
+            forwardPaginationToken, EventTimeline.FORWARDS,
+        );
     }
-    newTimeline.initialiseState(events);
 
     // make sure we set the pagination token before firing timelineReset,
     // otherwise clients which start back-paginating will fail, and then get
     // stuck without realising that they *can* back-paginate.
     newTimeline.setPaginationToken(backPaginationToken, EventTimeline.BACKWARDS);
 
+    // Now we can swap the live timeline to the new one.
     this._liveTimeline = newTimeline;
-    this.emit("Room.timelineReset", this.room, this);
+    this.emit("Room.timelineReset", this.room, this, resetAllTimelines);
 };
 
 /**
@@ -195,7 +206,7 @@ EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flu
  * the given event, or null if unknown
  */
 EventTimelineSet.prototype.getTimelineForEvent = function(eventId) {
-    var res = this._eventIdToTimeline[eventId];
+    const res = this._eventIdToTimeline[eventId];
     return (res === undefined) ? null : res;
 };
 
@@ -206,12 +217,13 @@ EventTimelineSet.prototype.getTimelineForEvent = function(eventId) {
  * @return {?module:models/event~MatrixEvent} the given event, or undefined if unknown
  */
 EventTimelineSet.prototype.findEventById = function(eventId) {
-    var tl = this.getTimelineForEvent(eventId);
+    const tl = this.getTimelineForEvent(eventId);
     if (!tl) {
         return undefined;
     }
-    return utils.findElement(tl.getEvents(),
-                             function(ev) { return ev.getId() == eventId; });
+    return utils.findElement(tl.getEvents(), function(ev) {
+        return ev.getId() == eventId;
+    });
 };
 
 /**
@@ -226,7 +238,7 @@ EventTimelineSet.prototype.addTimeline = function() {
                         " it.");
     }
 
-    var timeline = new EventTimeline(this);
+    const timeline = new EventTimeline(this);
     this._timelines.push(timeline);
     return timeline;
 };
@@ -255,14 +267,14 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
                                               timeline, paginationToken) {
     if (!timeline) {
         throw new Error(
-            "'timeline' not specified for EventTimelineSet.addEventsToTimeline"
+            "'timeline' not specified for EventTimelineSet.addEventsToTimeline",
         );
     }
 
     if (!toStartOfTimeline && timeline == this._liveTimeline) {
         throw new Error(
             "EventTimelineSet.addEventsToTimeline cannot be used for adding events to " +
-            "the live timeline - use Room.addLiveEvents instead"
+            "the live timeline - use Room.addLiveEvents instead",
         );
     }
 
@@ -273,9 +285,9 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
         }
     }
 
-    var direction = toStartOfTimeline ? EventTimeline.BACKWARDS :
+    const direction = toStartOfTimeline ? EventTimeline.BACKWARDS :
         EventTimeline.FORWARDS;
-    var inverseDirection = toStartOfTimeline ? EventTimeline.FORWARDS :
+    const inverseDirection = toStartOfTimeline ? EventTimeline.FORWARDS :
         EventTimeline.BACKWARDS;
 
     // Adding events to timelines can be quite complicated. The following
@@ -347,13 +359,13 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
     // paginating with the same token, we might as well use the new pagination
     // token in the hope that we eventually work our way out of the mess.
 
-    var didUpdate = false;
-    var lastEventWasNew = false;
-    for (var i = 0; i < events.length; i++) {
-        var event = events[i];
-        var eventId = event.getId();
+    let didUpdate = false;
+    let lastEventWasNew = false;
+    for (let i = 0; i < events.length; i++) {
+        const event = events[i];
+        const eventId = event.getId();
 
-        var existingTimeline = this._eventIdToTimeline[eventId];
+        const existingTimeline = this._eventIdToTimeline[eventId];
 
         if (!existingTimeline) {
             // we don't know about this event yet. Just add it to the timeline.
@@ -370,7 +382,7 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
             continue;
         }
 
-        var neighbour = timeline.getNeighbouringTimeline(direction);
+        const neighbour = timeline.getNeighbouringTimeline(direction);
         if (neighbour) {
             // this timeline already has a neighbour in the relevant direction;
             // let's assume the timelines are already correctly linked up, and
@@ -418,25 +430,25 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
  */
 EventTimelineSet.prototype.addLiveEvent = function(event, duplicateStrategy) {
     if (this._filter) {
-        var events = this._filter.filterRoomTimeline([event]);
+        const events = this._filter.filterRoomTimeline([event]);
         if (!events.length) {
             return;
         }
     }
 
-    var timeline = this._eventIdToTimeline[event.getId()];
+    const timeline = this._eventIdToTimeline[event.getId()];
     if (timeline) {
         if (duplicateStrategy === "replace") {
             debuglog("EventTimelineSet.addLiveEvent: replacing duplicate event " +
                      event.getId());
-            var tlEvents = timeline.getEvents();
-            for (var j = 0; j < tlEvents.length; j++) {
+            const tlEvents = timeline.getEvents();
+            for (let j = 0; j < tlEvents.length; j++) {
                 if (tlEvents[j].getId() === event.getId()) {
                     // still need to set the right metadata on this event
                     EventTimeline.setEventMetadata(
                         event,
                         timeline.getState(EventTimeline.FORWARDS),
-                        false
+                        false,
                     );
 
                     if (!tlEvents[j].encryptedType) {
@@ -471,11 +483,11 @@ EventTimelineSet.prototype.addLiveEvent = function(event, duplicateStrategy) {
  */
 EventTimelineSet.prototype.addEventToTimeline = function(event, timeline,
                                                          toStartOfTimeline) {
-    var eventId = event.getId();
+    const eventId = event.getId();
     timeline.addEvent(event, toStartOfTimeline);
     this._eventIdToTimeline[eventId] = timeline;
 
-    var data = {
+    const data = {
         timeline: timeline,
         liveEvent: !toStartOfTimeline && timeline == this._liveTimeline,
     };
@@ -496,7 +508,7 @@ EventTimelineSet.prototype.addEventToTimeline = function(event, timeline,
 EventTimelineSet.prototype.handleRemoteEcho = function(localEvent, oldEventId,
                                                         newEventId) {
     // XXX: why don't we infer newEventId from localEvent?
-    var existingTimeline = this._eventIdToTimeline[oldEventId];
+    const existingTimeline = this._eventIdToTimeline[oldEventId];
     if (existingTimeline) {
         delete this._eventIdToTimeline[oldEventId];
         this._eventIdToTimeline[newEventId] = existingTimeline;
@@ -505,8 +517,7 @@ EventTimelineSet.prototype.handleRemoteEcho = function(localEvent, oldEventId,
             if (this._filter.filterRoomTimeline([localEvent]).length) {
                 this.addEventToTimeline(localEvent, this._liveTimeline, false);
             }
-        }
-        else {
+        } else {
             this.addEventToTimeline(localEvent, this._liveTimeline, false);
         }
     }
@@ -521,15 +532,15 @@ EventTimelineSet.prototype.handleRemoteEcho = function(localEvent, oldEventId,
  * in this room.
  */
 EventTimelineSet.prototype.removeEvent = function(eventId) {
-    var timeline = this._eventIdToTimeline[eventId];
+    const timeline = this._eventIdToTimeline[eventId];
     if (!timeline) {
         return null;
     }
 
-    var removed = timeline.removeEvent(eventId);
+    const removed = timeline.removeEvent(eventId);
     if (removed) {
         delete this._eventIdToTimeline[eventId];
-        var data = {
+        const data = {
             timeline: timeline,
         };
         this.emit("Room.timeline", removed, this.room, undefined, true, data);
@@ -555,8 +566,8 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
         return 0;
     }
 
-    var timeline1 = this._eventIdToTimeline[eventId1];
-    var timeline2 = this._eventIdToTimeline[eventId2];
+    const timeline1 = this._eventIdToTimeline[eventId1];
+    const timeline2 = this._eventIdToTimeline[eventId2];
 
     if (timeline1 === undefined) {
         return null;
@@ -568,11 +579,11 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
     if (timeline1 === timeline2) {
         // both events are in the same timeline - figure out their
         // relative indices
-        var idx1, idx2;
-        var events = timeline1.getEvents();
-        for (var idx = 0; idx < events.length &&
+        let idx1, idx2;
+        const events = timeline1.getEvents();
+        for (let idx = 0; idx < events.length &&
              (idx1 === undefined || idx2 === undefined); idx++) {
-            var evId = events[idx].getId();
+            const evId = events[idx].getId();
             if (evId == eventId1) {
                 idx1 = idx;
             }
@@ -587,7 +598,7 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
     // linkedlist to see which comes first.
 
     // first work forwards from timeline1
-    var tl = timeline1;
+    let tl = timeline1;
     while (tl) {
         if (tl === timeline2) {
             // timeline1 is before timeline2
@@ -652,4 +663,5 @@ module.exports = EventTimelineSet;
  * @event module:client~MatrixClient#"Room.timelineReset"
  * @param {Room} room The room whose live timeline was reset, if any
  * @param {EventTimelineSet} timelineSet timelineSet room whose live timeline was reset
+ * @param {boolean} resetAllTimelines True if all timelines were reset.
  */

src/models/event-timeline.js

@@ -1,12 +1,25 @@
+/*
+Copyright 2016, 2017 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 "use strict";
 
 /**
  * @module models/event-timeline
  */
 
-var RoomState = require("./room-state");
-var utils = require("../utils");
-var MatrixEvent = require("./event").MatrixEvent;
+const RoomState = require("./room-state");
 
 /**
  * Construct a new EventTimeline
@@ -73,17 +86,70 @@ EventTimeline.prototype.initialiseState = function(stateEvents) {
         throw new Error("Cannot initialise state after events are added");
     }
 
-    // we deep-copy the events here, in case they get changed later - we don't
-    // want changes to the start state leaking through to the end state.
-    var oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEvents.map(function(mxEvent) { return mxEvent.event; })
-        ), function(ev) { return new MatrixEvent(ev); });
+    // We previously deep copied events here and used different copies in
+    // the oldState and state events: this decision seems to date back
+    // quite a way and was apparently made to fix a bug where modifications
+    // made to the start state leaked through to the end state.
+    // This really shouldn't be possible though: the events themselves should
+    // not change. Duplicating the events uses a lot of extra memory,
+    // so we now no longer do it. To assert that they really do never change,
+    // freeze them! Note that we can't do this for events in general:
+    // although it looks like the only things preventing us are the
+    // 'status' flag, forwardLooking (which is only set once when adding to the
+    // timeline) and possibly the sender (which seems like it should never be
+    // reset but in practice causes a lot of the tests to break).
+    for (const e of stateEvents) {
+        Object.freeze(e);
+    }
 
-    this._startState.setStateEvents(oldStateEvents);
+    this._startState.setStateEvents(stateEvents);
     this._endState.setStateEvents(stateEvents);
 };
 
+/**
+ * Forks the (live) timeline, taking ownership of the existing directional state of this timeline.
+ * All attached listeners will keep receiving state updates from the new live timeline state.
+ * The end state of this timeline gets replaced with an independent copy of the current RoomState,
+ * and will need a new pagination token if it ever needs to paginate forwards.
+
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.forkLive = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    // Now clobber the end state of the new live timeline with that from the
+    // previous live timeline. It will be identical except that we'll keep
+    // using the same RoomMember objects for the 'live' set of members with any
+    // listeners still attached
+    timeline._endState = forkState;
+    // Firstly, we just stole the current timeline's end state, so it needs a new one.
+    // Make an immutable copy of the state so back pagination will get the correct sentinels.
+    this._endState = forkState.clone();
+    return timeline;
+};
+
+/**
+ * Creates an independent timeline, inheriting the directional state from this timeline.
+ *
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.fork = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    timeline._endState = forkState.clone();
+    return timeline;
+};
+
 /**
  * Get the ID of the room for this timeline
  * @return {string} room ID
@@ -232,13 +298,12 @@ EventTimeline.prototype.setNeighbouringTimeline = function(neighbour, direction)
  * @param {boolean}  atStart     true to insert new event at the start
  */
 EventTimeline.prototype.addEvent = function(event, atStart) {
-    var stateContext = atStart ? this._startState : this._endState;
+    const stateContext = atStart ? this._startState : this._endState;
 
     // only call setEventMetadata on the unfiltered timelineSets
-    var timelineSet = this.getTimelineSet();
+    const timelineSet = this.getTimelineSet();
     if (timelineSet.room &&
-        timelineSet.room.getUnfilteredTimelineSet() === timelineSet)
-    {
+        timelineSet.room.getUnfilteredTimelineSet() === timelineSet) {
         EventTimeline.setEventMetadata(event, stateContext, atStart);
 
         // modify state
@@ -260,7 +325,7 @@ EventTimeline.prototype.addEvent = function(event, atStart) {
         }
     }
 
-    var insertIndex;
+    let insertIndex;
 
     if (atStart) {
         insertIndex = 0;
@@ -284,11 +349,11 @@ EventTimeline.prototype.addEvent = function(event, atStart) {
 EventTimeline.setEventMetadata = function(event, stateContext, toStartOfTimeline) {
     // set sender and target properties
     event.sender = stateContext.getSentinelMember(
-        event.getSender()
+        event.getSender(),
     );
     if (event.getType() === "m.room.member") {
         event.target = stateContext.getSentinelMember(
-            event.getStateKey()
+            event.getStateKey(),
         );
     }
     if (event.isState()) {
@@ -309,8 +374,8 @@ EventTimeline.setEventMetadata = function(event, stateContext, toStartOfTimeline
  * @return {?MatrixEvent} removed event, or null if not found
  */
 EventTimeline.prototype.removeEvent = function(eventId) {
-    for (var i = this._events.length - 1; i >= 0; i--) {
-        var ev = this._events[i];
+    for (let i = this._events.length - 1; i >= 0; i--) {
+        const ev = this._events[i];
         if (ev.getId() == eventId) {
             this._events.splice(i, 1);
             if (i < this._baseIndex) {

src/models/event.js

@@ -21,9 +21,9 @@ limitations under the License.
  * @module models/event
  */
 
-var EventEmitter = require("events").EventEmitter;
-
-var utils = require('../utils.js');
+import Promise from 'bluebird';
+import {EventEmitter} from 'events';
+import utils from '../utils.js';
 
 /**
  * Enum for event statuses.
@@ -49,6 +49,8 @@ module.exports.EventStatus = {
     CANCELLED: "cancelled",
 };
 
+const interns = {};
+
 /**
  * Construct a Matrix Event object
  * @constructor
@@ -61,29 +63,83 @@ module.exports.EventStatus = {
  * from changes to event JSON between Matrix versions.
  *
  * @prop {RoomMember} sender The room member who sent this event, or null e.g.
- * this is a presence event.
+ * this is a presence event. This is only guaranteed to be set for events that
+ * appear in a timeline, ie. do not guarantee that it will be set on state
+ * events.
  * @prop {RoomMember} target The room member who is the target of this event, e.g.
  * the invitee, the person being banned, etc.
  * @prop {EventStatus} status The sending status of the event.
+ * @prop {Error} error most recent error associated with sending the event, if any
  * @prop {boolean} forwardLooking True if this event is 'forward looking', meaning
  * that getDirectionalContent() will return event.content and not event.prev_content.
  * Default: true. <strong>This property is experimental and may change.</strong>
  */
 module.exports.MatrixEvent = function MatrixEvent(
-    event
+    event,
 ) {
+    // intern the values of matrix events to force share strings and reduce the
+    // amount of needless string duplication. This can save moderate amounts of
+    // memory (~10% on a 350MB heap).
+    // 'membership' at the event level (rather than the content level) is a legacy
+    // field that Riot never otherwise looks at, but it will still take up a lot
+    // of space if we don't intern it.
+    ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
+        if (!event[prop]) {
+            return;
+        }
+        if (!interns[event[prop]]) {
+            interns[event[prop]] = event[prop];
+        }
+        event[prop] = interns[event[prop]];
+    });
+
+    ["membership", "avatar_url", "displayname"].forEach((prop) => {
+        if (!event.content || !event.content[prop]) {
+            return;
+        }
+        if (!interns[event.content[prop]]) {
+            interns[event.content[prop]] = event.content[prop];
+        }
+        event.content[prop] = interns[event.content[prop]];
+    });
+
     this.event = event || {};
+
     this.sender = null;
     this.target = null;
     this.status = null;
+    this.error = null;
     this.forwardLooking = true;
     this._pushActions = null;
-    this._date = this.event.origin_server_ts ?
-        new Date(this.event.origin_server_ts) : null;
 
     this._clearEvent = {};
-    this._keysProved = {};
-    this._keysClaimed = {};
+
+    /* curve25519 key which we believe belongs to the sender of the event. See
+     * getSenderKey()
+     */
+    this._senderCurve25519Key = null;
+
+    /* ed25519 key which the sender of this event (for olm) or the creator of
+     * the megolm session (for megolm) claims to own. See getClaimedEd25519Key()
+     */
+    this._claimedEd25519Key = null;
+
+    /* curve25519 keys of devices involved in telling us about the
+     * _senderCurve25519Key and _claimedEd25519Key.
+     * See getForwardingCurve25519KeyChain().
+     */
+    this._forwardingCurve25519KeyChain = [];
+
+    /* if we have a process decrypting this event, a Promise which resolves
+     * when it is finished. Normally null.
+     */
+    this._decryptionPromise = null;
+
+    /* flag to indicate if we should retry decrypting this event after the
+     * first attempt (eg, we have received new data which means that a second
+     * attempt may succeed)
+     */
+    this._retryDecryption = false;
 };
 utils.inherits(module.exports.MatrixEvent, EventEmitter);
 
@@ -149,7 +205,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Date} The event date, e.g. <code>new Date(1433502692297)</code>
      */
     getDate: function() {
-        return this._date;
+        return this.event.origin_server_ts ? new Date(this.event.origin_server_ts) : null;
     },
 
     /**
@@ -231,9 +287,18 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * <tt>"m.room.encrypted"</tt>
      *
      * @param {object} crypto_content raw 'content' for the encrypted event.
-     * @param {object} keys The local keys claimed and proved by this event.
+     *
+     * @param {string} senderCurve25519Key curve25519 key to record for the
+     *   sender of this event.
+     *   See {@link module:models/event.MatrixEvent#getSenderKey}.
+     *
+     * @param {string} claimedEd25519Key claimed ed25519 key to record for the
+     *   sender if this event.
+     *   See {@link module:models/event.MatrixEvent#getClaimedEd25519Key}
      */
-    makeEncrypted: function(crypto_type, crypto_content, keys) {
+    makeEncrypted: function(
+        crypto_type, crypto_content, senderCurve25519Key, claimedEd25519Key,
+    ) {
         // keep the plain-text data for 'view source'
         this._clearEvent = {
             type: this.event.type,
@@ -241,8 +306,187 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         };
         this.event.type = crypto_type;
         this.event.content = crypto_content;
-        this._keysProved = keys;
-        this._keysClaimed = keys;
+        this._senderCurve25519Key = senderCurve25519Key;
+        this._claimedEd25519Key = claimedEd25519Key;
+    },
+
+    /**
+     * Check if this event is currently being decrypted.
+     *
+     * @return {boolean} True if this event is currently being decrypted, else false.
+     */
+    isBeingDecrypted: function() {
+        return this._decryptionPromise != null;
+    },
+
+    /**
+     * Check if this event is an encrypted event which we failed to decrypt
+     *
+     * (This implies that we might retry decryption at some point in the future)
+     *
+     * @return {boolean} True if this event is an encrypted event which we
+     *     couldn't decrypt.
+     */
+    isDecryptionFailure: function() {
+        return this._clearEvent && this._clearEvent.content &&
+            this._clearEvent.content.msgtype === "m.bad.encrypted";
+    },
+
+    /**
+     * Start the process of trying to decrypt this event.
+     *
+     * (This is used within the SDK: it isn't intended for use by applications)
+     *
+     * @internal
+     *
+     * @param {module:crypto} crypto crypto module
+     *
+     * @returns {Promise} promise which resolves (to undefined) when the decryption
+     * attempt is completed.
+     */
+    attemptDecryption: async function(crypto) {
+        // start with a couple of sanity checks.
+        if (!this.isEncrypted()) {
+            throw new Error("Attempt to decrypt event which isn't encrypted");
+        }
+
+        if (
+            this._clearEvent && this._clearEvent.content &&
+                this._clearEvent.content.msgtype !== "m.bad.encrypted"
+        ) {
+            // we may want to just ignore this? let's start with rejecting it.
+            throw new Error(
+                "Attempt to decrypt event which has already been encrypted",
+            );
+        }
+
+        // if we already have a decryption attempt in progress, then it may
+        // fail because it was using outdated info. We now have reason to
+        // succeed where it failed before, but we don't want to have multiple
+        // attempts going at the same time, so just set a flag that says we have
+        // new info.
+        //
+        if (this._decryptionPromise) {
+            console.log(
+                `Event ${this.getId()} already being decrypted; queueing a retry`,
+            );
+            this._retryDecryption = true;
+            return this._decryptionPromise;
+        }
+
+        this._decryptionPromise = this._decryptionLoop(crypto);
+        return this._decryptionPromise;
+    },
+
+    /**
+     * Cancel any room key request for this event and resend another.
+     *
+     * @param {module:crypto} crypto crypto module
+     */
+    cancelAndResendKeyRequest: function(crypto) {
+        const wireContent = this.getWireContent();
+        crypto.cancelRoomKeyRequest({
+            algorithm: wireContent.algorithm,
+            room_id: this.getRoomId(),
+            session_id: wireContent.session_id,
+            sender_key: wireContent.sender_key,
+        }, true);
+    },
+
+    _decryptionLoop: async function(crypto) {
+        // make sure that this method never runs completely synchronously.
+        // (doing so would mean that we would clear _decryptionPromise *before*
+        // it is set in attemptDecryption - and hence end up with a stuck
+        // `_decryptionPromise`).
+        await Promise.resolve();
+
+        while (true) {
+            this._retryDecryption = false;
+
+            let res;
+            let err;
+            try {
+                if (!crypto) {
+                    res = this._badEncryptedMessage("Encryption not enabled");
+                } else {
+                    res = await crypto.decryptEvent(this);
+                }
+            } catch (e) {
+                if (e.name !== "DecryptionError") {
+                    // not a decryption error: log the whole exception as an error
+                    // (and don't bother with a retry)
+                    console.error(
+                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                    );
+                    this._decryptionPromise = null;
+                    this._retryDecryption = false;
+                    return;
+                }
+
+                err = e;
+
+                // see if we have a retry queued.
+                //
+                // NB: make sure to keep this check in the same tick of the
+                //   event loop as `_decryptionPromise = null` below - otherwise we
+                //   risk a race:
+                //
+                //   * A: we check _retryDecryption here and see that it is
+                //        false
+                //   * B: we get a second call to attemptDecryption, which sees
+                //        that _decryptionPromise is set so sets
+                //        _retryDecryption
+                //   * A: we continue below, clear _decryptionPromise, and
+                //        never do the retry.
+                //
+                if (this._retryDecryption) {
+                    // decryption error, but we have a retry queued.
+                    console.log(
+                        `Got error decrypting event (id=${this.getId()}: ` +
+                        `${e}), but retrying`,
+                    );
+                    continue;
+                }
+
+                // decryption error, no retries queued. Warn about the error and
+                // set it to m.bad.encrypted.
+                console.warn(
+                    `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
+                );
+
+                res = this._badEncryptedMessage(e.message);
+            }
+
+            // at this point, we've either successfully decrypted the event, or have given up
+            // (and set res to a 'badEncryptedMessage'). Either way, we can now set the
+            // cleartext of the event and raise Event.decrypted.
+            //
+            // make sure we clear '_decryptionPromise' before sending the 'Event.decrypted' event,
+            // otherwise the app will be confused to see `isBeingDecrypted` still set when
+            // there isn't an `Event.decrypted` on the way.
+            //
+            // see also notes on _retryDecryption above.
+            //
+            this._decryptionPromise = null;
+            this._retryDecryption = false;
+            this._setClearData(res);
+
+            this.emit("Event.decrypted", this, err);
+
+            return;
+        }
+    },
+
+    _badEncryptedMessage: function(reason) {
+        return {
+            clearEvent: {
+                type: "m.room.message",
+                content: {
+                    msgtype: "m.bad.encrypted",
+                    body: "** Unable to decrypt: " + reason + " **",
+                },
+            },
+        };
     },
 
     /**
@@ -254,20 +498,17 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      *
      * @fires module:models/event.MatrixEvent#"Event.decrypted"
      *
-     * @param {Object} clearEvent The plaintext payload for the event
-     *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
-     *
-     * @param {Object=} keysProved Keys owned by the sender of this event.
-     *    See {@link module:models/event.MatrixEvent#getKeysProved}.
-     *
-     * @param {Object=} keysClaimed Keys the sender of this event claims.
-     *    See {@link module:models/event.MatrixEvent#getKeysClaimed}.
+     * @param {module:crypto~EventDecryptionResult} decryptionResult
+     *     the decryption result, including the plaintext and some key info
      */
-    setClearData: function(clearEvent, keysProved, keysClaimed) {
-        this._clearEvent = clearEvent;
-        this._keysProved = keysProved || {};
-        this._keysClaimed = keysClaimed || {};
-        this.emit("Event.decrypted", this);
+    _setClearData: function(decryptionResult) {
+        this._clearEvent = decryptionResult.clearEvent;
+        this._senderCurve25519Key =
+            decryptionResult.senderCurve25519Key || null;
+        this._claimedEd25519Key =
+            decryptionResult.claimedEd25519Key || null;
+        this._forwardingCurve25519KeyChain =
+            decryptionResult.forwardingCurve25519KeyChain || [];
     },
 
     /**
@@ -279,37 +520,72 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     /**
-     * The curve25519 key that sent this event
+     * The curve25519 key for the device that we think sent this event
+     *
+     * For an Olm-encrypted event, this is inferred directly from the DH
+     * exchange at the start of the session: the curve25519 key is involved in
+     * the DH exchange, so only a device which holds the private part of that
+     * key can establish such a session.
+     *
+     * For a megolm-encrypted event, it is inferred from the Olm message which
+     * established the megolm session
+     *
      * @return {string}
      */
     getSenderKey: function() {
-        return this.getKeysProved().curve25519 || null;
-    },
-
-    /**
-     * The keys that must have been owned by the sender of this encrypted event.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     *
-     * @return {Object<string, string>}
-     */
-    getKeysProved: function() {
-        return this._keysProved;
+        return this._senderCurve25519Key;
     },
 
     /**
      * The additional keys the sender of this encrypted event claims to possess.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     * For example megolm messages don't claim keys directly, but instead
-     * inherit a claim from the olm message that established the session.
+     *
+     * Just a wrapper for #getClaimedEd25519Key (q.v.)
      *
      * @return {Object<string, string>}
      */
     getKeysClaimed: function() {
-        return this._keysClaimed;
+        return {
+            ed25519: this._claimedEd25519Key,
+        };
+    },
+
+    /**
+     * Get the ed25519 the sender of this event claims to own.
+     *
+     * For Olm messages, this claim is encoded directly in the plaintext of the
+     * event itself. For megolm messages, it is implied by the m.room_key event
+     * which established the megolm session.
+     *
+     * Until we download the device list of the sender, it's just a claim: the
+     * device list gives a proof that the owner of the curve25519 key used for
+     * this event (and returned by #getSenderKey) also owns the ed25519 key by
+     * signing the public curve25519 key with the ed25519 key.
+     *
+     * In general, applications should not use this method directly, but should
+     * instead use MatrixClient.getEventSenderDeviceInfo.
+     *
+     * @return {string}
+     */
+    getClaimedEd25519Key: function() {
+        return this._claimedEd25519Key;
+    },
+
+    /**
+     * Get the curve25519 keys of the devices which were involved in telling us
+     * about the claimedEd25519Key and sender curve25519 key.
+     *
+     * Normally this will be empty, but in the case of a forwarded megolm
+     * session, the sender keys are sent to us by another device (the forwarding
+     * device), which we need to trust to do this. In that case, the result will
+     * be a list consisting of one entry.
+     *
+     * If the device that sent us the key (A) got it from another device which
+     * it wasn't prepared to vouch for (B), the result will be [A, B]. And so on.
+     *
+     * @return {string[]} base64-encoded curve25519 keys, from oldest to newest.
+     */
+    getForwardingCurve25519KeyChain: function() {
+        return this._forwardingCurve25519KeyChain;
     },
 
     getUnsigned: function() {
@@ -340,18 +616,22 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         }
         this.event.unsigned.redacted_because = redaction_event.event;
 
-        var key;
+        let key;
         for (key in this.event) {
-            if (!this.event.hasOwnProperty(key)) { continue; }
+            if (!this.event.hasOwnProperty(key)) {
+                continue;
+            }
             if (!_REDACT_KEEP_KEY_MAP[key]) {
                 delete this.event[key];
             }
         }
 
-        var keeps = _REDACT_KEEP_CONTENT_MAP[this.getType()] || {};
-        var content = this.getContent();
+        const keeps = _REDACT_KEEP_CONTENT_MAP[this.getType()] || {};
+        const content = this.getContent();
         for (key in content) {
-            if (!content.hasOwnProperty(key)) { continue; }
+            if (!content.hasOwnProperty(key)) {
+                continue;
+            }
             if (!keeps[key]) {
                 delete content[key];
             }
@@ -393,40 +673,28 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         this.event = event;
         // successfully sent.
         this.status = null;
-        this._date = new Date(this.event.origin_server_ts);
-     }
+     },
 });
 
 
-/* http://matrix.org/docs/spec/r0.0.1/client_server.html#redactions says:
+/* _REDACT_KEEP_KEY_MAP gives the keys we keep when an event is redacted
  *
- * the server should strip off any keys not in the following list:
- *    event_id
- *    type
- *    room_id
- *    user_id
- *    state_key
- *    prev_state
- *    content
- *    [we keep 'unsigned' as well, since that is created by the local server]
+ * This is specified here:
+ *  http://matrix.org/speculator/spec/HEAD/client_server/unstable.html#redactions
  *
- * The content object should also be stripped of all keys, unless it is one of
- * one of the following event types:
- *    m.room.member allows key membership
- *    m.room.create allows key creator
- *    m.room.join_rules allows key join_rule
- *    m.room.power_levels allows keys ban, events, events_default, kick,
- *        redact, state_default, users, users_default.
- *    m.room.aliases allows key aliases
+ * Also:
+ *  - We keep 'unsigned' since that is created by the local server
+ *  - We keep user_id for backwards-compat with v1
  */
-// a map giving the keys we keep when an event is redacted
-var _REDACT_KEEP_KEY_MAP = [
-    'event_id', 'type', 'room_id', 'user_id', 'state_key', 'prev_state',
-    'content', 'unsigned',
-].reduce(function(ret, val) { ret[val] = 1; return ret; }, {});
+const _REDACT_KEEP_KEY_MAP = [
+    'event_id', 'type', 'room_id', 'user_id', 'sender', 'state_key', 'prev_state',
+    'content', 'unsigned', 'origin_server_ts',
+].reduce(function(ret, val) {
+    ret[val] = 1; return ret;
+}, {});
 
 // a map from event type to the .content keys we keep when an event is redacted
-var _REDACT_KEEP_CONTENT_MAP = {
+const _REDACT_KEEP_CONTENT_MAP = {
     'm.room.member': {'membership': 1},
     'm.room.create': {'creator': 1},
     'm.room.join_rules': {'join_rule': 1},
@@ -445,4 +713,7 @@ var _REDACT_KEEP_CONTENT_MAP = {
  *
  * @param {module:models/event.MatrixEvent} event
  *    The matrix event which has been decrypted
+ * @param {module:crypto/algorithms/base.DecryptionError?} err
+ *    The error that occured during decryption, or `undefined` if no
+ *    error occured.
  */

src/models/group.js

@@ -0,0 +1,95 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module models/group
+ */
+const EventEmitter = require("events").EventEmitter;
+
+const utils = require("../utils");
+
+/**
+ * Construct a new Group.
+ *
+ * @param {string} groupId The ID of this group.
+ *
+ * @prop {string} groupId The ID of this group.
+ * @prop {string} name The human-readable display name for this group.
+ * @prop {string} avatarUrl The mxc URL for this group's avatar.
+ * @prop {string} myMembership The logged in user's membership of this group
+ * @prop {Object} inviter Infomation about the user who invited the logged in user
+ *       to the group, if myMembership is 'invite'.
+ * @prop {string} inviter.userId The user ID of the inviter
+ */
+function Group(groupId) {
+    this.groupId = groupId;
+    this.name = null;
+    this.avatarUrl = null;
+    this.myMembership = null;
+    this.inviter = null;
+}
+utils.inherits(Group, EventEmitter);
+
+Group.prototype.setProfile = function(name, avatarUrl) {
+    if (this.name === name && this.avatarUrl === avatarUrl) return;
+
+    this.name = name || this.groupId;
+    this.avatarUrl = avatarUrl;
+
+    this.emit("Group.profile", this);
+};
+
+Group.prototype.setMyMembership = function(membership) {
+    if (this.myMembership === membership) return;
+
+    this.myMembership = membership;
+
+    this.emit("Group.myMembership", this);
+};
+
+/**
+ * Sets the 'inviter' property. This does not emit an event (the inviter
+ * will only change when the user is revited / reinvited to a room),
+ * so set this before setting myMembership.
+ * @param {Object} inviter Infomation about who invited us to the room
+ */
+Group.prototype.setInviter = function(inviter) {
+    this.inviter = inviter;
+};
+
+module.exports = Group;
+
+/**
+ * Fires whenever a group's profile information is updated.
+ * This means the 'name' and 'avatarUrl' properties.
+ * @event module:client~MatrixClient#"Group.profile"
+ * @param {Group} group The group whose profile was updated.
+ * @example
+ * matrixClient.on("Group.profile", function(group){
+ *   var name = group.name;
+ * });
+ */
+
+/**
+ * Fires whenever the logged in user's membership status of
+ * the group is updated.
+ * @event module:client~MatrixClient#"Group.myMembership"
+ * @param {Group} group The group in which the user's membership changed
+ * @example
+ * matrixClient.on("Group.myMembership", function(group){
+ *   var myMembership = group.myMembership;
+ * });
+ */

src/models/room-member.js

@@ -17,10 +17,10 @@ limitations under the License.
 /**
  * @module models/room-member
  */
-var EventEmitter = require("events").EventEmitter;
-var ContentRepo = require("../content-repo");
+const EventEmitter = require("events").EventEmitter;
+const ContentRepo = require("../content-repo");
 
-var utils = require("../utils");
+const utils = require("../utils");
 
 /**
  * Construct a new room member.
@@ -33,7 +33,10 @@ var utils = require("../utils");
  * @prop {string} roomId The room ID for this member.
  * @prop {string} userId The user ID of this member.
  * @prop {boolean} typing True if the room member is currently typing.
- * @prop {string} name The human-readable name for this room member.
+ * @prop {string} name The human-readable name for this room member. This will be
+ * disambiguated with a suffix of " (@user_id:matrix.org)" if another member shares the
+ * same displayname.
+ * @prop {string} rawDisplayName The ambiguous displayname of this room member.
  * @prop {Number} powerLevel The power level for this room member.
  * @prop {Number} powerLevelNorm The normalised power level (0-100) for this
  * room member.
@@ -47,17 +50,35 @@ function RoomMember(roomId, userId) {
     this.userId = userId;
     this.typing = false;
     this.name = userId;
+    this.rawDisplayName = userId;
     this.powerLevel = 0;
     this.powerLevelNorm = 0;
     this.user = null;
     this.membership = null;
     this.events = {
-        member: null
+        member: null,
     };
+    this._isOutOfBand = false;
     this._updateModifiedTime();
 }
 utils.inherits(RoomMember, EventEmitter);
 
+/**
+ * Mark the member as coming from a channel that is not sync
+ */
+RoomMember.prototype.markOutOfBand = function() {
+    this._isOutOfBand = true;
+};
+
+/**
+ * @return {bool} does the member come from a channel that is not sync?
+ * This is used to store the member seperately
+ * from the sync state so it available across browser sessions.
+ */
+RoomMember.prototype.isOutOfBand = function() {
+    return this._isOutOfBand;
+};
+
 /**
  * Update this room member's membership event. May fire "RoomMember.name" if
  * this event updates this member's name.
@@ -71,13 +92,21 @@ RoomMember.prototype.setMembershipEvent = function(event, roomState) {
     if (event.getType() !== "m.room.member") {
         return;
     }
+
+    this._isOutOfBand = false;
+
     this.events.member = event;
 
-    var oldMembership = this.membership;
+    const oldMembership = this.membership;
     this.membership = event.getDirectionalContent().membership;
 
-    var oldName = this.name;
-    this.name = calculateDisplayName(this, event, roomState);
+    const oldName = this.name;
+    this.name = calculateDisplayName(
+        this.userId,
+        event.getDirectionalContent().displayname,
+        roomState);
+
+    this.rawDisplayName = event.getDirectionalContent().displayname || this.userId;
     if (oldMembership !== this.membership) {
         this._updateModifiedTime();
         this.emit("RoomMember.membership", event, this, oldMembership);
@@ -99,17 +128,20 @@ RoomMember.prototype.setPowerLevelEvent = function(powerLevelEvent) {
     if (powerLevelEvent.getType() !== "m.room.power_levels") {
         return;
     }
-    var maxLevel = powerLevelEvent.getContent().users_default || 0;
-    utils.forEach(utils.values(powerLevelEvent.getContent().users), function(lvl) {
+
+    const evContent = powerLevelEvent.getDirectionalContent();
+
+    let maxLevel = evContent.users_default || 0;
+    utils.forEach(utils.values(evContent.users), function(lvl) {
         maxLevel = Math.max(maxLevel, lvl);
     });
-    var oldPowerLevel = this.powerLevel;
-    var oldPowerLevelNorm = this.powerLevelNorm;
+    const oldPowerLevel = this.powerLevel;
+    const oldPowerLevelNorm = this.powerLevelNorm;
 
-    if (powerLevelEvent.getContent().users[this.userId] !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users[this.userId];
-    } else if (powerLevelEvent.getContent().users_default !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users_default;
+    if (evContent.users && evContent.users[this.userId] !== undefined) {
+        this.powerLevel = evContent.users[this.userId];
+    } else if (evContent.users_default !== undefined) {
+        this.powerLevel = evContent.users_default;
     } else {
         this.powerLevel = 0;
     }
@@ -136,9 +168,9 @@ RoomMember.prototype.setTypingEvent = function(event) {
     if (event.getType() !== "m.typing") {
         return;
     }
-    var oldTyping = this.typing;
+    const oldTyping = this.typing;
     this.typing = false;
-    var typingList = event.getContent().user_ids;
+    const typingList = event.getContent().user_ids;
     if (!utils.isArray(typingList)) {
         // malformed event :/ bail early. TODO: whine?
         return;
@@ -169,6 +201,44 @@ RoomMember.prototype.getLastModifiedTime = function() {
     return this._modified;
 };
 
+
+RoomMember.prototype.isKicked = function() {
+    return this.membership === "leave" &&
+        this.events.member.getSender() !== this.events.member.getStateKey();
+};
+
+/**
+ * If this member was invited with the is_direct flag set, return
+ * the user that invited this member
+ * @return {string} user id of the inviter
+ */
+RoomMember.prototype.getDMInviter = function() {
+    // when not available because that room state hasn't been loaded in,
+    // we don't really know, but more likely to not be a direct chat
+    if (this.events.member) {
+        // TODO: persist the is_direct flag on the member as more member events
+        //       come in caused by displayName changes.
+
+        // the is_direct flag is set on the invite member event.
+        // This is copied on the prev_content section of the join member event
+        // when the invite is accepted.
+
+        const memberEvent = this.events.member;
+        let memberContent = memberEvent.getContent();
+        let inviteSender = memberEvent.getSender();
+
+        if (memberContent.membership === "join") {
+            memberContent = memberEvent.getPrevContent();
+            inviteSender = memberEvent.getUnsigned().prev_sender;
+        }
+
+        if (memberContent.membership === "invite" && memberContent.is_direct) {
+            return inviteSender;
+        }
+    }
+};
+
+
 /**
  * Get the avatar URL for a room member.
  * @param {string} baseUrl The base homeserver URL See
@@ -189,30 +259,42 @@ RoomMember.prototype.getLastModifiedTime = function() {
  */
 RoomMember.prototype.getAvatarUrl =
         function(baseUrl, width, height, resizeMethod, allowDefault, allowDirectLinks) {
-    if (allowDefault === undefined) { allowDefault = true; }
-    if (!this.events.member && !allowDefault) {
+    if (allowDefault === undefined) {
+        allowDefault = true;
+    }
+
+    const rawUrl = this.getMxcAvatarUrl();
+
+    if (!rawUrl && !allowDefault) {
         return null;
     }
-    var rawUrl = this.events.member ? this.events.member.getContent().avatar_url : null;
-    var httpUrl = ContentRepo.getHttpUriForMxc(
-        baseUrl, rawUrl, width, height, resizeMethod, allowDirectLinks
+    const httpUrl = ContentRepo.getHttpUriForMxc(
+        baseUrl, rawUrl, width, height, resizeMethod, allowDirectLinks,
     );
     if (httpUrl) {
         return httpUrl;
-    }
-    else if (allowDefault) {
+    } else if (allowDefault) {
         return ContentRepo.getIdenticonUri(
-            baseUrl, this.userId, width, height
+            baseUrl, this.userId, width, height,
         );
     }
     return null;
 };
+/**
+ * get the mxc avatar url, either from a state event, or from a lazily loaded member
+ * @return {string} the mxc avatar url
+ */
+RoomMember.prototype.getMxcAvatarUrl = function() {
+    if(this.events.member) {
+        return this.events.member.getDirectionalContent().avatar_url;
+    } else if(this.user) {
+        return this.user.avatarUrl;
+    }
+    return null;
+};
 
-function calculateDisplayName(member, event, roomState) {
-    var displayName = event.getDirectionalContent().displayname;
-    var selfUserId = member.userId;
-
-    if (!displayName) {
+function calculateDisplayName(selfUserId, displayName, roomState) {
+    if (!displayName || displayName === selfUserId) {
         return selfUserId;
     }
 
@@ -220,11 +302,26 @@ function calculateDisplayName(member, event, roomState) {
         return displayName;
     }
 
-    var userIds = roomState.getUserIdsWithDisplayName(displayName);
-    var otherUsers = userIds.filter(function(u) {
-        return u !== selfUserId;
-    });
-    if (otherUsers.length > 0) {
+    // First check if the displayname is something we consider truthy
+    // after stripping it of zero width characters and padding spaces
+    const strippedDisplayName = utils.removeHiddenChars(displayName);
+    if (!strippedDisplayName) {
+        return selfUserId;
+    }
+
+    // Next check if the name contains something that look like a mxid
+    // If it does, it may be someone trying to impersonate someone else
+    // Show full mxid in this case
+    // Also show mxid if there are other people with the same displayname
+    // ignoring any zero width chars (unicode 200B-200D)
+    // if their displayname is made up of just zero width chars, show full mxid
+    let disambiguate = /@.+:.+/.test(displayName);
+    if (!disambiguate) {
+        const userIds = roomState.getUserIdsWithDisplayName(strippedDisplayName);
+        disambiguate = userIds.some((u) => u !== selfUserId);
+    }
+
+    if (disambiguate) {
         return displayName + " (" + selfUserId + ")";
     }
     return displayName;

src/models/room-state.js

@@ -17,23 +17,51 @@ limitations under the License.
 /**
  * @module models/room-state
  */
-var EventEmitter = require("events").EventEmitter;
+const EventEmitter = require("events").EventEmitter;
 
-var utils = require("../utils");
-var RoomMember = require("./room-member");
+const utils = require("../utils");
+const RoomMember = require("./room-member");
+
+// possible statuses for out-of-band member loading
+const OOB_STATUS_NOTSTARTED = 1;
+const OOB_STATUS_INPROGRESS = 2;
+const OOB_STATUS_FINISHED = 3;
 
 /**
  * Construct room state.
+ *
+ * Room State represents the state of the room at a given point.
+ * It can be mutated by adding state events to it.
+ * There are two types of room member associated with a state event:
+ * normal member objects (accessed via getMember/getMembers) which mutate
+ * with the state to represent the current state of that room/user, eg.
+ * the object returned by getMember('@bob:example.com') will mutate to
+ * get a different display name if Bob later changes his display name
+ * in the room.
+ * There are also 'sentinel' members (accessed via getSentinelMember).
+ * These also represent the state of room members at the point in time
+ * represented by the RoomState object, but unlike objects from getMember,
+ * sentinel objects will always represent the room state as at the time
+ * getSentinelMember was called, so if Bob subsequently changes his display
+ * name, a room member object previously acquired with getSentinelMember
+ * will still have his old display name. Calling getSentinelMember again
+ * after the display name change will return a new RoomMember object
+ * with Bob's new display name.
+ *
  * @constructor
  * @param {?string} roomId Optional. The ID of the room which has this state.
  * If none is specified it just tracks paginationTokens, useful for notifTimelineSet
+ * @param {?object} oobMemberFlags Optional. The state of loading out of bound members.
+ * As the timeline might get reset while they are loading, this state needs to be inherited
+ * and shared when the room state is cloned for the new timeline.
+ * This should only be passed from clone.
  * @prop {Object.<string, RoomMember>} members The room member dictionary, keyed
  * on the user's ID.
  * @prop {Object.<string, Object.<string, MatrixEvent>>} events The state
  * events dictionary, keyed on the event type and then the state_key value.
  * @prop {string} paginationToken The pagination token for this state.
  */
-function RoomState(roomId) {
+function RoomState(roomId, oobMemberFlags = undefined) {
     this.roomId = roomId;
     this.members = {
         // userId: RoomMember
@@ -50,9 +78,74 @@ function RoomState(roomId) {
     this._displayNameToUserIds = {};
     this._userIdsToDisplayNames = {};
     this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite
+    this._joinedMemberCount = null; // cache of the number of joined members
+    // joined members count from summary api
+    // once set, we know the server supports the summary api
+    // and we should only trust that
+    // we could also only trust that before OOB members
+    // are loaded but doesn't seem worth the hassle atm
+    this._summaryJoinedMemberCount = null;
+    // same for invited member count
+    this._invitedMemberCount = null;
+    this._summaryInvitedMemberCount = null;
+
+    if (!oobMemberFlags) {
+        oobMemberFlags = {
+            status: OOB_STATUS_NOTSTARTED,
+        };
+    }
+    this._oobMemberFlags = oobMemberFlags;
 }
 utils.inherits(RoomState, EventEmitter);
 
+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+RoomState.prototype.getJoinedMemberCount = function() {
+    if (this._summaryJoinedMemberCount !== null) {
+        return this._summaryJoinedMemberCount;
+    }
+    if (this._joinedMemberCount === null) {
+        this._joinedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'join' ? count + 1 : count;
+        }, 0);
+    }
+    return this._joinedMemberCount;
+};
+
+/**
+ * Set the joined member count explicitly (like from summary part of the sync response)
+ * @param {number} count the amount of joined members
+ */
+RoomState.prototype.setJoinedMemberCount = function(count) {
+    this._summaryJoinedMemberCount = count;
+};
+/**
+ * Returns the number of invited members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite'
+ */
+RoomState.prototype.getInvitedMemberCount = function() {
+    if (this._summaryInvitedMemberCount !== null) {
+        return this._summaryInvitedMemberCount;
+    }
+    if (this._invitedMemberCount === null) {
+        this._invitedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'invite' ? count + 1 : count;
+        }, 0);
+    }
+    return this._invitedMemberCount;
+};
+
+/**
+ * Set the amount of invited members in this room
+ * @param {number} count the amount of invited members
+ */
+RoomState.prototype.setInvitedMemberCount = function(count) {
+    this._summaryInvitedMemberCount = count;
+};
+
 /**
  * Get all RoomMembers in this room.
  * @return {Array<RoomMember>} A list of RoomMembers.
@@ -80,7 +173,18 @@ RoomState.prototype.getMember = function(userId) {
  * @return {RoomMember} The member or null if they do not exist.
  */
 RoomState.prototype.getSentinelMember = function(userId) {
-    return this._sentinels[userId] || null;
+    if (!userId) return null;
+    let sentinel = this._sentinels[userId];
+
+    if (sentinel === undefined) {
+        sentinel = new RoomMember(this.roomId, userId);
+        const member = this.members[userId];
+        if (member) {
+            sentinel.setMembershipEvent(member.events.member, this);
+        }
+        this._sentinels[userId] = sentinel;
+    }
+    return sentinel;
 };
 
 /**
@@ -100,10 +204,71 @@ RoomState.prototype.getStateEvents = function(eventType, stateKey) {
     if (stateKey === undefined) { // return all values
         return utils.values(this.events[eventType]);
     }
-    var event = this.events[eventType][stateKey];
+    const event = this.events[eventType][stateKey];
     return event ? event : null;
 };
 
+/**
+ * Creates a copy of this room state so that mutations to either won't affect the other.
+ * @return {RoomState} the copy of the room state
+ */
+RoomState.prototype.clone = function() {
+    const copy = new RoomState(this.roomId, this._oobMemberFlags);
+
+    // Ugly hack: because setStateEvents will mark
+    // members as susperseding future out of bound members
+    // if loading is in progress (through _oobMemberFlags)
+    // since these are not new members, we're merely copying them
+    // set the status to not started
+    // after copying, we set back the status
+    const status = this._oobMemberFlags.status;
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+
+    Object.values(this.events).forEach((eventsByStateKey) => {
+        const eventsForType = Object.values(eventsByStateKey);
+        copy.setStateEvents(eventsForType);
+    });
+
+    // Ugly hack: see above
+    this._oobMemberFlags.status = status;
+
+    if (this._summaryInvitedMemberCount !== null) {
+        copy.setInvitedMemberCount(this.getInvitedMemberCount());
+    }
+    if (this._summaryJoinedMemberCount !== null) {
+        copy.setJoinedMemberCount(this.getJoinedMemberCount());
+    }
+
+    // copy out of band flags if needed
+    if (this._oobMemberFlags.status == OOB_STATUS_FINISHED) {
+        // copy markOutOfBand flags
+        this.getMembers().forEach((member) => {
+            if (member.isOutOfBand()) {
+                const copyMember = copy.getMember(member.userId);
+                copyMember.markOutOfBand();
+            }
+        });
+    }
+
+    return copy;
+};
+
+/**
+ * Add previously unknown state events.
+ * When lazy loading members while back-paginating,
+ * the relevant room state for the timeline chunk at the end
+ * of the chunk can be set with this method.
+ * @param {MatrixEvent[]} events state events to prepend
+ */
+RoomState.prototype.setUnknownStateEvents = function(events) {
+    const unknownStateEvents = events.filter((event) => {
+        return this.events[event.getType()] === undefined ||
+            this.events[event.getType()][event.getStateKey()] === undefined;
+    });
+
+    this.setStateEvents(unknownStateEvents);
+};
+
 /**
  * Add an array of one or more state MatrixEvents, overwriting
  * any existing state with the same {type, stateKey} tuple. Will fire
@@ -115,21 +280,22 @@ RoomState.prototype.getStateEvents = function(eventType, stateKey) {
  * @fires module:client~MatrixClient#event:"RoomState.events"
  */
 RoomState.prototype.setStateEvents = function(stateEvents) {
-    var self = this;
+    const self = this;
     this._updateModifiedTime();
 
     // update the core event dict
     utils.forEach(stateEvents, function(event) {
-        if (event.getRoomId() !== self.roomId) { return; }
-        if (!event.isState()) { return; }
-
-        if (self.events[event.getType()] === undefined) {
-            self.events[event.getType()] = {};
+        if (event.getRoomId() !== self.roomId) {
+            return;
         }
-        self.events[event.getType()][event.getStateKey()] = event;
+        if (!event.isState()) {
+            return;
+        }
+
+        self._setStateEvent(event);
         if (event.getType() === "m.room.member") {
             _updateDisplayNameCache(
-                self, event.getStateKey(), event.getContent().displayname
+                self, event.getStateKey(), event.getContent().displayname,
             );
             _updateThirdPartyTokenCache(self, event);
         }
@@ -141,18 +307,21 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
     // the given array (e.g. disambiguating display names in one go to do both
     // clashing names rather than progressively which only catches 1 of them).
     utils.forEach(stateEvents, function(event) {
-        if (event.getRoomId() !== self.roomId) { return; }
-        if (!event.isState()) { return; }
+        if (event.getRoomId() !== self.roomId) {
+            return;
+        }
+        if (!event.isState()) {
+            return;
+        }
 
         if (event.getType() === "m.room.member") {
-            var userId = event.getStateKey();
+            const userId = event.getStateKey();
 
             // leave events apparently elide the displayname or avatar_url,
             // so let's fake one up so that we don't leak user ids
             // into the timeline
             if (event.getContent().membership === "leave" ||
-                event.getContent().membership === "ban")
-            {
+                event.getContent().membership === "ban") {
                 event.getContent().avatar_url =
                     event.getContent().avatar_url ||
                     event.getPrevContent().avatar_url;
@@ -161,40 +330,158 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                     event.getPrevContent().displayname;
             }
 
-            var member = self.members[userId];
-            if (!member) {
-                member = new RoomMember(event.getRoomId(), userId);
-                self.emit("RoomState.newMember", event, self, member);
-            }
-            // Add a new sentinel for this change. We apply the same
-            // operations to both sentinel and member rather than deep copying
-            // so we don't make assumptions about the properties of RoomMember
-            // (e.g. and manage to break it because deep copying doesn't do
-            // everything).
-            var sentinel = new RoomMember(event.getRoomId(), userId);
-            utils.forEach([member, sentinel], function(roomMember) {
-                roomMember.setMembershipEvent(event, self);
-                // this member may have a power level already, so set it.
-                var pwrLvlEvent = self.getStateEvents("m.room.power_levels", "");
-                if (pwrLvlEvent) {
-                    roomMember.setPowerLevelEvent(pwrLvlEvent);
-                }
-            });
+            const member = self._getOrCreateMember(userId, event);
+            member.setMembershipEvent(event, self);
 
-            self._sentinels[userId] = sentinel;
-            self.members[userId] = member;
+            self._updateMember(member);
             self.emit("RoomState.members", event, self, member);
-        }
-        else if (event.getType() === "m.room.power_levels") {
-            var members = utils.values(self.members);
+        } else if (event.getType() === "m.room.power_levels") {
+            const members = utils.values(self.members);
             utils.forEach(members, function(member) {
                 member.setPowerLevelEvent(event);
                 self.emit("RoomState.members", event, self, member);
             });
+
+            // assume all our sentinels are now out-of-date
+            self._sentinels = {};
         }
     });
 };
 
+/**
+ * Looks up a member by the given userId, and if it doesn't exist,
+ * create it and emit the `RoomState.newMember` event.
+ * This method makes sure the member is added to the members dictionary
+ * before emitting, as this is done from setStateEvents and _setOutOfBandMember.
+ * @param {string} userId the id of the user to look up
+ * @param {MatrixEvent} event the membership event for the (new) member. Used to emit.
+ * @fires module:client~MatrixClient#event:"RoomState.newMember"
+ * @returns {RoomMember} the member, existing or newly created.
+ */
+RoomState.prototype._getOrCreateMember = function(userId, event) {
+    let member = this.members[userId];
+    if (!member) {
+        member = new RoomMember(this.roomId, userId);
+        // add member to members before emitting any events,
+        // as event handlers often lookup the member
+        this.members[userId] = member;
+        this.emit("RoomState.newMember", event, this, member);
+    }
+    return member;
+};
+
+RoomState.prototype._setStateEvent = function(event) {
+    if (this.events[event.getType()] === undefined) {
+        this.events[event.getType()] = {};
+    }
+    this.events[event.getType()][event.getStateKey()] = event;
+};
+
+RoomState.prototype._updateMember = function(member) {
+    // this member may have a power level already, so set it.
+    const pwrLvlEvent = this.getStateEvents("m.room.power_levels", "");
+    if (pwrLvlEvent) {
+        member.setPowerLevelEvent(pwrLvlEvent);
+    }
+
+    // blow away the sentinel which is now outdated
+    delete this._sentinels[member.userId];
+
+    this.members[member.userId] = member;
+    this._joinedMemberCount = null;
+    this._invitedMemberCount = null;
+};
+
+/**
+ * Get the out-of-band members loading state, whether loading is needed or not.
+ * Note that loading might be in progress and hence isn't needed.
+ * @return {bool} whether or not the members of this room need to be loaded
+ */
+RoomState.prototype.needsOutOfBandMembers = function() {
+    return this._oobMemberFlags.status === OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Mark this room state as waiting for out-of-band members,
+ * ensuring it doesn't ask for them to be requested again
+ * through needsOutOfBandMembers
+ */
+RoomState.prototype.markOutOfBandMembersStarted = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_NOTSTARTED) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_INPROGRESS;
+};
+
+/**
+ * Mark this room state as having failed to fetch out-of-band members
+ */
+RoomState.prototype.markOutOfBandMembersFailed = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Clears the loaded out-of-band members
+ */
+RoomState.prototype.clearOutOfBandMembers = function() {
+    let count = 0;
+    Object.keys(this.members).forEach((userId) => {
+        const member = this.members[userId];
+        if (member.isOutOfBand()) {
+            ++count;
+            delete this.members[userId];
+        }
+    });
+    console.log(`LL: RoomState removed ${count} members...`);
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Sets the loaded out-of-band members.
+ * @param {MatrixEvent[]} stateEvents array of membership state events
+ */
+RoomState.prototype.setOutOfBandMembers = function(stateEvents) {
+    console.log(`LL: RoomState about to set ${stateEvents.length} OOB members ...`);
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    console.log(`LL: RoomState put in OOB_STATUS_FINISHED state ...`);
+    this._oobMemberFlags.status = OOB_STATUS_FINISHED;
+    stateEvents.forEach((e) => this._setOutOfBandMember(e));
+};
+
+/**
+ * Sets a single out of band member, used by both setOutOfBandMembers and clone
+ * @param {MatrixEvent} stateEvent membership state event
+ */
+RoomState.prototype._setOutOfBandMember = function(stateEvent) {
+    if (stateEvent.getType() !== 'm.room.member') {
+        return;
+    }
+    const userId = stateEvent.getStateKey();
+    const existingMember = this.getMember(userId);
+    // never replace members received as part of the sync
+    if (existingMember && !existingMember.isOutOfBand()) {
+        return;
+    }
+
+    const member = this._getOrCreateMember(userId, stateEvent);
+    member.setMembershipEvent(stateEvent, this);
+    // needed to know which members need to be stored seperately
+    // as they are not part of the sync accumulator
+    // this is cleared by setMembershipEvent so when it's updated through /sync
+    member.markOutOfBand();
+
+    _updateDisplayNameCache(this, member.userId, member.name);
+
+    this._setStateEvent(stateEvent);
+    this._updateMember(member);
+    this.emit("RoomState.members", stateEvent, this, member);
+};
+
 /**
  * Set the current typing event for this room.
  * @param {MatrixEvent} event The typing event
@@ -240,6 +527,49 @@ RoomState.prototype.getUserIdsWithDisplayName = function(displayName) {
     return this._displayNameToUserIds[displayName] || [];
 };
 
+/**
+ * Returns true if userId is in room, event is not redacted and either sender of
+ * mxEvent or has power level sufficient to redact events other than their own.
+ * @param {MatrixEvent} mxEvent The event to test permission for
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given used ID can redact given event
+ */
+RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
+    const member = this.getMember(userId);
+    if (!member || member.membership === 'leave') return false;
+
+    if (mxEvent.status || mxEvent.isRedacted()) return false;
+
+    // The user may have been the sender, but they can't redact their own message
+    // if redactions are blocked.
+    const canRedact = this.maySendEvent("m.room.redaction", userId);
+    if (mxEvent.getSender() === userId) return canRedact;
+
+    return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
+};
+
+/**
+ * Returns true if the given power level is sufficient for action
+ * @param {string} action The type of power level to check
+ * @param {number} powerLevel The power level of the member
+ * @return {boolean} true if the given power level is sufficient
+ */
+RoomState.prototype._hasSufficientPowerLevelFor = function(action, powerLevel) {
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let powerLevels = {};
+    if (powerLevelsEvent) {
+        powerLevels = powerLevelsEvent.getContent();
+    }
+
+    let requiredLevel = 50;
+    if (utils.isNumber(powerLevels[action])) {
+        requiredLevel = powerLevels[action];
+    }
+
+    return powerLevel >= requiredLevel;
+};
+
 /**
  * Short-form for maySendEvent('m.room.message', userId)
  * @param {string} userId The user ID of the user to test permission for
@@ -263,7 +593,6 @@ RoomState.prototype.maySendEvent = function(eventType, userId) {
     return this._maySendEventOfType(eventType, userId, false);
 };
 
-
 /**
  * Returns true if the given MatrixClient has permission to send a state
  * event of type `stateEventType` into this room.
@@ -306,35 +635,70 @@ RoomState.prototype.maySendStateEvent = function(stateEventType, userId) {
  *                        according to the room's state.
  */
 RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
-    var member = this.getMember(userId);
-    if (!member || member.membership == 'leave') { return false; }
+    const power_levels_event = this.getStateEvents('m.room.power_levels', '');
 
-    var power_levels_event = this.getStateEvents('m.room.power_levels', '');
+    let power_levels;
+    let events_levels = {};
 
-    var power_levels;
-    var events_levels = {};
-
-    var state_default = 0;
-    var events_default = 0;
+    let state_default = 0;
+    let events_default = 0;
+    let powerLevel = 0;
     if (power_levels_event) {
         power_levels = power_levels_event.getContent();
         events_levels = power_levels.events || {};
 
-        if (power_levels.state_default !== undefined) {
+        if (Number.isFinite(power_levels.state_default)) {
             state_default = power_levels.state_default;
         } else {
             state_default = 50;
         }
-        if (power_levels.events_default !== undefined) {
+
+        const userPowerLevel = power_levels.users && power_levels.users[userId];
+        if (Number.isFinite(userPowerLevel)) {
+            powerLevel = userPowerLevel;
+        } else if(Number.isFinite(power_levels.users_default)) {
+            powerLevel = power_levels.users_default;
+        }
+
+        if (Number.isFinite(power_levels.events_default)) {
             events_default = power_levels.events_default;
         }
     }
 
-    var required_level = state ? state_default : events_default;
-    if (events_levels[eventType] !== undefined) {
+    let required_level = state ? state_default : events_default;
+    if (Number.isFinite(events_levels[eventType])) {
         required_level = events_levels[eventType];
     }
-    return member.powerLevel >= required_level;
+    return powerLevel >= required_level;
+};
+
+/**
+ * Returns true if the given user ID has permission to trigger notification
+ * of type `notifLevelKey`
+ * @param {string} notifLevelKey The level of notification to test (eg. 'room')
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given user ID has permission to trigger a
+ *                        notification of this type.
+ */
+RoomState.prototype.mayTriggerNotifOfType = function(notifLevelKey, userId) {
+    const member = this.getMember(userId);
+    if (!member) {
+        return false;
+    }
+
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let notifLevel = 50;
+    if (
+        powerLevelsEvent &&
+        powerLevelsEvent.getContent() &&
+        powerLevelsEvent.getContent().notifications &&
+        utils.isNumber(powerLevelsEvent.getContent().notifications[notifLevelKey])
+    ) {
+        notifLevel = powerLevelsEvent.getContent().notifications[notifLevelKey];
+    }
+
+    return member.powerLevel >= notifLevel;
 };
 
 /**
@@ -347,12 +711,12 @@ function _updateThirdPartyTokenCache(roomState, memberEvent) {
     if (!memberEvent.getContent().third_party_invite) {
         return;
     }
-    var token = (memberEvent.getContent().third_party_invite.signed || {}).token;
+    const token = (memberEvent.getContent().third_party_invite.signed || {}).token;
     if (!token) {
         return;
     }
-    var threePidInvite = roomState.getStateEvents(
-        "m.room.third_party_invite", token
+    const threePidInvite = roomState.getStateEvents(
+        "m.room.third_party_invite", token,
     );
     if (!threePidInvite) {
         return;
@@ -361,29 +725,33 @@ function _updateThirdPartyTokenCache(roomState, memberEvent) {
 }
 
 function _updateDisplayNameCache(roomState, userId, displayName) {
-    var oldName = roomState._userIdsToDisplayNames[userId];
+    const oldName = roomState._userIdsToDisplayNames[userId];
     delete roomState._userIdsToDisplayNames[userId];
     if (oldName) {
         // Remove the old name from the cache.
         // We clobber the user_id > name lookup but the name -> [user_id] lookup
         // means we need to remove that user ID from that array rather than nuking
         // the lot.
-        var existingUserIds = roomState._displayNameToUserIds[oldName] || [];
-        for (var i = 0; i < existingUserIds.length; i++) {
-            if (existingUserIds[i] === userId) {
-                // remove this user ID from this array
-                existingUserIds.splice(i, 1);
-                i--;
-            }
+        const strippedOldName = utils.removeHiddenChars(oldName);
+
+        const existingUserIds = roomState._displayNameToUserIds[strippedOldName];
+        if (existingUserIds) {
+            // remove this user ID from this array
+            const filteredUserIDs = existingUserIds.filter((id) => id !== userId);
+            roomState._displayNameToUserIds[strippedOldName] = filteredUserIDs;
         }
-        roomState._displayNameToUserIds[oldName] = existingUserIds;
     }
 
     roomState._userIdsToDisplayNames[userId] = displayName;
-    if (!roomState._displayNameToUserIds[displayName]) {
-        roomState._displayNameToUserIds[displayName] = [];
+
+    const strippedDisplayname = displayName && utils.removeHiddenChars(displayName);
+    // an empty stripped displayname (undefined/'') will be set to MXID in room-member.js
+    if (strippedDisplayname) {
+        if (!roomState._displayNameToUserIds[strippedDisplayname]) {
+            roomState._displayNameToUserIds[strippedDisplayname] = [];
+        }
+        roomState._displayNameToUserIds[strippedDisplayname].push(userId);
     }
-    roomState._displayNameToUserIds[displayName].push(userId);
 }
 
 /**
@@ -413,7 +781,8 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
 
  /**
  * Fires whenever a member is added to the members dictionary. The RoomMember
- * will not be fully populated yet (e.g. no membership state).
+ * will not be fully populated yet (e.g. no membership state) but will already
+ * be available in the members dictionary.
  * @event module:client~MatrixClient#"RoomState.newMember"
  * @param {MatrixEvent} event The matrix event which caused this event to fire.
  * @param {RoomState} state The room state whose RoomState.members dictionary

src/models/search-result.js

@@ -19,8 +19,8 @@ limitations under the License.
  * @module models/search-result
  */
 
-var EventContext = require("./event-context");
-var utils = require("../utils");
+const EventContext = require("./event-context");
+const utils = require("../utils");
 
 /**
  * Construct a new SearchResult
@@ -45,11 +45,11 @@ function SearchResult(rank, eventContext) {
  */
 
 SearchResult.fromJson = function(jsonObj, eventMapper) {
-    var jsonContext = jsonObj.context || {};
-    var events_before = jsonContext.events_before || [];
-    var events_after = jsonContext.events_after || [];
+    const jsonContext = jsonObj.context || {};
+    const events_before = jsonContext.events_before || [];
+    const events_after = jsonContext.events_after || [];
 
-    var context = new EventContext(eventMapper(jsonObj.result));
+    const context = new EventContext(eventMapper(jsonObj.result));
 
     context.setPaginateToken(jsonContext.start, true);
     context.addEvents(utils.map(events_before, eventMapper), true);

src/models/user.js

@@ -17,8 +17,8 @@ limitations under the License.
 /**
  * @module models/user
  */
- var EventEmitter = require("events").EventEmitter;
- var utils = require("../utils");
+ const EventEmitter = require("events").EventEmitter;
+ const utils = require("../utils");
 
 /**
  * Construct a new User. A User must have an ID and can optionally have extra
@@ -54,7 +54,7 @@ function User(userId) {
     this.currentlyActive = false;
     this.events = {
         presence: null,
-        profile: null
+        profile: null,
     };
     this._updateModifiedTime();
 }
@@ -73,26 +73,23 @@ User.prototype.setPresenceEvent = function(event) {
     if (event.getType() !== "m.presence") {
         return;
     }
-    var firstFire = this.events.presence === null;
+    const firstFire = this.events.presence === null;
     this.events.presence = event;
 
-    var eventsToFire = [];
+    const eventsToFire = [];
     if (event.getContent().presence !== this.presence || firstFire) {
         eventsToFire.push("User.presence");
     }
     if (event.getContent().avatar_url &&
-        event.getContent().avatar_url !== this.avatarUrl)
-    {
+        event.getContent().avatar_url !== this.avatarUrl) {
         eventsToFire.push("User.avatarUrl");
     }
     if (event.getContent().displayname &&
-        event.getContent().displayname !== this.displayName)
-    {
+        event.getContent().displayname !== this.displayName) {
         eventsToFire.push("User.displayName");
     }
     if (event.getContent().currently_active !== undefined &&
-        event.getContent().currently_active !== this.currentlyActive)
-    {
+        event.getContent().currently_active !== this.currentlyActive) {
         eventsToFire.push("User.currentlyActive");
     }
 
@@ -114,7 +111,7 @@ User.prototype.setPresenceEvent = function(event) {
 
     this._updateModifiedTime();
 
-    for (var i = 0; i < eventsToFire.length; i++) {
+    for (let i = 0; i < eventsToFire.length; i++) {
         this.emit(eventsToFire[i], event, this);
     }
 };
@@ -125,7 +122,7 @@ User.prototype.setPresenceEvent = function(event) {
  * @param {string} name The new display name.
  */
 User.prototype.setDisplayName = function(name) {
-    var oldName = this.displayName;
+    const oldName = this.displayName;
     this.displayName = name;
     if (name !== oldName) {
         this._updateModifiedTime();
@@ -149,7 +146,7 @@ User.prototype.setRawDisplayName = function(name) {
  * @param {string} url The new avatar URL.
  */
 User.prototype.setAvatarUrl = function(url) {
-    var oldUrl = this.avatarUrl;
+    const oldUrl = this.avatarUrl;
     this.avatarUrl = url;
     if (url !== oldUrl) {
         this._updateModifiedTime();

src/pushprocessor.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,32 +18,41 @@ limitations under the License.
  * @module pushprocessor
  */
 
+const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
+
 /**
  * Construct a Push Processor.
  * @constructor
  * @param {Object} client The Matrix client object to use
  */
 function PushProcessor(client) {
-    var escapeRegExp = function(string) {
+    const escapeRegExp = function(string) {
         return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
     };
 
-    var matchingRuleFromKindSet = function(ev, kindset, device) {
-        var rulekinds_in_order = ['override', 'content', 'room', 'sender', 'underride'];
-        for (var ruleKindIndex = 0;
-                ruleKindIndex < rulekinds_in_order.length;
+    const cachedGlobToRegex = {
+        // $glob: RegExp,
+    };
+
+    const matchingRuleFromKindSet = (ev, kindset, device) => {
+        for (let ruleKindIndex = 0;
+                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                 ++ruleKindIndex) {
-            var kind = rulekinds_in_order[ruleKindIndex];
-            var ruleset = kindset[kind];
+            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
+            const ruleset = kindset[kind];
 
-            for (var ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
-                var rule = ruleset[ruleIndex];
-                if (!rule.enabled) { continue; }
+            for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
+                const rule = ruleset[ruleIndex];
+                if (!rule.enabled) {
+                    continue;
+                }
 
-                var rawrule = templateRuleToRaw(kind, rule, device);
-                if (!rawrule) { continue; }
+                const rawrule = templateRuleToRaw(kind, rule, device);
+                if (!rawrule) {
+                    continue;
+                }
 
-                if (ruleMatchesEvent(rawrule, ev)) {
+                if (this.ruleMatchesEvent(rawrule, ev)) {
                     rule.kind = kind;
                     return rule;
                 }
@@ -51,11 +61,11 @@ function PushProcessor(client) {
         return null;
     };
 
-    var templateRuleToRaw = function(kind, tprule, device) {
-        var rawrule = {
+    const templateRuleToRaw = function(kind, tprule, device) {
+        const rawrule = {
             'rule_id': tprule.rule_id,
             'actions': tprule.actions,
-            'conditions': []
+            'conditions': [],
         };
         switch (kind) {
             case 'underride':
@@ -63,77 +73,100 @@ function PushProcessor(client) {
                 rawrule.conditions = tprule.conditions;
                 break;
             case 'room':
-                if (!tprule.rule_id) { return null; }
+                if (!tprule.rule_id) {
+                    return null;
+                }
                 rawrule.conditions.push({
                     'kind': 'event_match',
                     'key': 'room_id',
-                    'pattern': tprule.rule_id
+                    'value': tprule.rule_id,
                 });
                 break;
             case 'sender':
-                if (!tprule.rule_id) { return null; }
+                if (!tprule.rule_id) {
+                    return null;
+                }
                 rawrule.conditions.push({
                     'kind': 'event_match',
                     'key': 'user_id',
-                    'pattern': tprule.rule_id
+                    'value': tprule.rule_id,
                 });
                 break;
             case 'content':
-                if (!tprule.pattern) { return null; }
+                if (!tprule.pattern) {
+                    return null;
+                }
                 rawrule.conditions.push({
                     'kind': 'event_match',
                     'key': 'content.body',
-                    'pattern': tprule.pattern
+                    'pattern': tprule.pattern,
                 });
                 break;
         }
         if (device) {
             rawrule.conditions.push({
                 'kind': 'device',
-                'profile_tag': device
+                'profile_tag': device,
             });
         }
         return rawrule;
     };
 
-    var ruleMatchesEvent = function(rule, ev) {
-        var ret = true;
-        for (var i = 0; i < rule.conditions.length; ++i) {
-            var cond = rule.conditions[i];
-            ret &= eventFulfillsCondition(cond, ev);
-        }
-        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
-        return ret;
-    };
-
-    var eventFulfillsCondition = function(cond, ev) {
-        var condition_functions = {
+    const eventFulfillsCondition = function(cond, ev) {
+        const condition_functions = {
             "event_match": eventFulfillsEventMatchCondition,
             "device": eventFulfillsDeviceCondition,
             "contains_display_name": eventFulfillsDisplayNameCondition,
-            "room_member_count": eventFulfillsRoomMemberCountCondition
+            "room_member_count": eventFulfillsRoomMemberCountCondition,
+            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
         };
         if (condition_functions[cond.kind]) {
             return condition_functions[cond.kind](cond, ev);
         }
-        return true;
+        // unknown conditions: we previously matched all unknown conditions,
+        // but given that rules can be added to the base rules on a server,
+        // it's probably better to not match unknown conditions.
+        return false;
     };
 
-    var eventFulfillsRoomMemberCountCondition = function(cond, ev) {
-        if (!cond.is) { return false; }
+    const eventFulfillsSenderNotifPermCondition = function(cond, ev) {
+        const notifLevelKey = cond['key'];
+        if (!notifLevelKey) {
+            return false;
+        }
 
-        var room = client.getRoom(ev.getRoomId());
-        if (!room || !room.currentState || !room.currentState.members) { return false; }
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState) {
+            return false;
+        }
 
-        var memberCount = Object.keys(room.currentState.members).filter(function(m) {
-            return room.currentState.members[m].membership == 'join';
-        }).length;
+        // Note that this should not be the current state of the room but the state at
+        // the point the event is in the DAG. Unfortunately the js-sdk does not store
+        // this.
+        return room.currentState.mayTriggerNotifOfType(notifLevelKey, ev.getSender());
+    };
 
-        var m = cond.is.match(/^([=<>]*)([0-9]*)$/);
-        if (!m) { return false; }
-        var ineq = m[1];
-        var rhs = parseInt(m[2]);
-        if (isNaN(rhs)) { return false; }
+    const eventFulfillsRoomMemberCountCondition = function(cond, ev) {
+        if (!cond.is) {
+            return false;
+        }
+
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState || !room.currentState.members) {
+            return false;
+        }
+
+        const memberCount = room.currentState.getJoinedMemberCount();
+
+        const m = cond.is.match(/^([=<>]*)([0-9]*)$/);
+        if (!m) {
+            return false;
+        }
+        const ineq = m[1];
+        const rhs = parseInt(m[2]);
+        if (isNaN(rhs)) {
+            return false;
+        }
         switch (ineq) {
             case '':
             case '==':
@@ -151,64 +184,88 @@ function PushProcessor(client) {
         }
     };
 
-    var eventFulfillsDisplayNameCondition = function(cond, ev) {
-        var content = ev.getContent();
+    const eventFulfillsDisplayNameCondition = function(cond, ev) {
+        const content = ev.getContent();
         if (!content || !content.body || typeof content.body != 'string') {
             return false;
         }
 
-        var room = client.getRoom(ev.getRoomId());
+        const room = client.getRoom(ev.getRoomId());
         if (!room || !room.currentState || !room.currentState.members ||
-            !room.currentState.getMember(client.credentials.userId)) { return false; }
+            !room.currentState.getMember(client.credentials.userId)) {
+            return false;
+        }
 
-        var displayName = room.currentState.getMember(client.credentials.userId).name;
+        const displayName = room.currentState.getMember(client.credentials.userId).name;
 
         // N.B. we can't use \b as it chokes on unicode. however \W seems to be okay
         // as shorthand for [^0-9A-Za-z_].
-        var pat = new RegExp("(^|\\W)" + escapeRegExp(displayName) + "(\\W|$)", 'i');
+        const pat = new RegExp("(^|\\W)" + escapeRegExp(displayName) + "(\\W|$)", 'i');
         return content.body.search(pat) > -1;
     };
 
-    var eventFulfillsDeviceCondition = function(cond, ev) {
+    const eventFulfillsDeviceCondition = function(cond, ev) {
         return false; // XXX: Allow a profile tag to be set for the web client instance
     };
 
-    var eventFulfillsEventMatchCondition = function(cond, ev) {
-        var val = valueForDottedKey(cond.key, ev);
-        if (!val || typeof val != 'string') { return false; }
-
-        var pat;
-        if (cond.key == 'content.body') {
-            pat = '(^|\\W)' + globToRegexp(cond.pattern) + '(\\W|$)';
-        } else {
-            pat = '^' + globToRegexp(cond.pattern) + '$';
+    const eventFulfillsEventMatchCondition = function(cond, ev) {
+        if (!cond.key) {
+            return false;
         }
-        var regex = new RegExp(pat, 'i');
+
+        const val = valueForDottedKey(cond.key, ev);
+        if (!val || typeof val != 'string') {
+            return false;
+        }
+
+        if (cond.value) {
+            return cond.value === val;
+        }
+
+        let regex;
+
+        if (cond.key == 'content.body') {
+            regex = createCachedRegex('(^|\\W)', cond.pattern, '(\\W|$)');
+        } else {
+            regex = createCachedRegex('^', cond.pattern, '$');
+        }
+
         return !!val.match(regex);
     };
 
-    var globToRegexp = function(glob) {
+    const createCachedRegex = function(prefix, glob, suffix) {
+        if (cachedGlobToRegex[glob]) {
+            return cachedGlobToRegex[glob];
+        }
+        cachedGlobToRegex[glob] = new RegExp(
+            prefix + globToRegexp(glob) + suffix,
+            'i', // Case insensitive
+        );
+        return cachedGlobToRegex[glob];
+    };
+
+    const globToRegexp = function(glob) {
         // From
         // https://github.com/matrix-org/synapse/blob/abbee6b29be80a77e05730707602f3bbfc3f38cb/synapse/push/__init__.py#L132
         // Because micromatch is about 130KB with dependencies,
         // and minimatch is not much better.
-        var pat = escapeRegExp(glob);
-        pat = pat.replace(/\\\*/, '.*');
-        pat = pat.replace(/\?/, '.');
-        pat = pat.replace(/\\\[(!|)(.*)\\]/, function(match, p1, p2, offset, string) {
-            var first = p1 && '^' || '';
-            var second = p2.replace(/\\\-/, '-');
+        let pat = escapeRegExp(glob);
+        pat = pat.replace(/\\\*/g, '.*');
+        pat = pat.replace(/\?/g, '.');
+        pat = pat.replace(/\\\[(!|)(.*)\\]/g, function(match, p1, p2, offset, string) {
+            const first = p1 && '^' || '';
+            const second = p2.replace(/\\\-/, '-');
             return '[' + first + second + ']';
         });
         return pat;
     };
 
-    var valueForDottedKey = function(key, ev) {
-        var parts = key.split('.');
-        var val;
+    const valueForDottedKey = function(key, ev) {
+        const parts = key.split('.');
+        let val;
 
         // special-case the first component to deal with encrypted messages
-        var firstPart = parts[0];
+        const firstPart = parts[0];
         if (firstPart == 'content') {
             val = ev.getContent();
             parts.shift();
@@ -221,33 +278,43 @@ function PushProcessor(client) {
         }
 
         while (parts.length > 0) {
-            var thispart = parts.shift();
-            if (!val[thispart]) { return null; }
+            const thispart = parts.shift();
+            if (!val[thispart]) {
+                return null;
+            }
             val = val[thispart];
         }
         return val;
     };
 
-    var matchingRuleForEventWithRulesets = function(ev, rulesets) {
-        if (!rulesets || !rulesets.device) { return null; }
-        if (ev.getSender() == client.credentials.userId) { return null; }
+    const matchingRuleForEventWithRulesets = function(ev, rulesets) {
+        if (!rulesets || !rulesets.device) {
+            return null;
+        }
+        if (ev.getSender() == client.credentials.userId) {
+            return null;
+        }
 
-        var allDevNames = Object.keys(rulesets.device);
-        for (var i = 0; i < allDevNames.length; ++i) {
-            var devname = allDevNames[i];
-            var devrules = rulesets.device[devname];
+        const allDevNames = Object.keys(rulesets.device);
+        for (let i = 0; i < allDevNames.length; ++i) {
+            const devname = allDevNames[i];
+            const devrules = rulesets.device[devname];
 
-            var matchingRule = matchingRuleFromKindSet(devrules, devname);
-            if (matchingRule) { return matchingRule; }
+            const matchingRule = matchingRuleFromKindSet(devrules, devname);
+            if (matchingRule) {
+                return matchingRule;
+            }
         }
         return matchingRuleFromKindSet(ev, rulesets.global);
     };
 
-    var pushActionsForEventAndRulesets = function(ev, rulesets) {
-        var rule = matchingRuleForEventWithRulesets(ev, rulesets);
-        if (!rule) { return {}; }
+    const pushActionsForEventAndRulesets = function(ev, rulesets) {
+        const rule = matchingRuleForEventWithRulesets(ev, rulesets);
+        if (!rule) {
+            return {};
+        }
 
-        var actionObj = PushProcessor.actionListToActionsObject(rule.actions);
+        const actionObj = PushProcessor.actionListToActionsObject(rule.actions);
 
         // Some actions are implicit in some situations: we add those here
         if (actionObj.tweaks.highlight === undefined) {
@@ -259,6 +326,17 @@ function PushProcessor(client) {
         return actionObj;
     };
 
+    this.ruleMatchesEvent = function(rule, ev) {
+        let ret = true;
+        for (let i = 0; i < rule.conditions.length; ++i) {
+            const cond = rule.conditions[i];
+            ret &= eventFulfillsCondition(cond, ev);
+        }
+        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
+        return ret;
+    };
+
+
     /**
      * Get the user's push actions for the given event
      *
@@ -269,6 +347,27 @@ function PushProcessor(client) {
     this.actionsForEvent = function(ev) {
         return pushActionsForEventAndRulesets(ev, client.pushRules);
     };
+
+    /**
+     * Get one of the users push rules by its ID
+     *
+     * @param {string} ruleId The ID of the rule to search for
+     * @return {object} The push rule, or null if no such rule was found
+     */
+    this.getPushRuleById = function(ruleId) {
+        for (const scope of ['device', 'global']) {
+            if (client.pushRules[scope] === undefined) continue;
+
+            for (const kind of RULEKINDS_IN_ORDER) {
+                if (client.pushRules[scope][kind] === undefined) continue;
+
+                for (const rule of client.pushRules[scope][kind]) {
+                    if (rule.rule_id === ruleId) return rule;
+                }
+            }
+        }
+        return null;
+    };
 }
 
 /**
@@ -280,13 +379,15 @@ function PushProcessor(client) {
  * @return {object} A object with key 'notify' (true or false) and an object of actions
  */
 PushProcessor.actionListToActionsObject = function(actionlist) {
-    var actionobj = { 'notify': false, 'tweaks': {} };
-    for (var i = 0; i < actionlist.length; ++i) {
-        var action = actionlist[i];
+    const actionobj = { 'notify': false, 'tweaks': {} };
+    for (let i = 0; i < actionlist.length; ++i) {
+        const action = actionlist[i];
         if (action === 'notify') {
             actionobj.notify = true;
         } else if (typeof action === 'object') {
-            if (action.value === undefined) { action.value = true; }
+            if (action.value === undefined) {
+                action.value = true;
+            }
             actionobj.tweaks[action.set_tweak] = action.value;
         }
     }

src/realtime-callbacks.js

@@ -27,20 +27,20 @@ limitations under the License.
 
 // we schedule a callback at least this often, to check if we've missed out on
 // some wall-clock time due to being suspended.
-var TIMER_CHECK_PERIOD_MS = 1000;
+const TIMER_CHECK_PERIOD_MS = 1000;
 
 // counter, for making up ids to return from setTimeout
-var _count = 0;
+let _count = 0;
 
 // the key for our callback with the real global.setTimeout
-var _realCallbackKey;
+let _realCallbackKey;
 
 // a sorted list of the callbacks to be run.
 // each is an object with keys [runAt, func, params, key].
-var _callbackList = [];
+const _callbackList = [];
 
 // var debuglog = console.log.bind(console);
-var debuglog = function() {};
+const debuglog = function() {};
 
 /**
  * Replace the function used by this module to get the current time.
@@ -54,7 +54,7 @@ var debuglog = function() {};
 module.exports.setNow = function(f) {
     _now = f || Date.now;
 };
-var _now = Date.now;
+let _now = Date.now;
 
 /**
  * reimplementation of window.setTimeout, which will call the callback if
@@ -72,12 +72,12 @@ module.exports.setTimeout = function(func, delayMs) {
         delayMs = 0;
     }
 
-    var params = Array.prototype.slice.call(arguments, 2);
-    var runAt = _now() + delayMs;
-    var key = _count++;
+    const params = Array.prototype.slice.call(arguments, 2);
+    const runAt = _now() + delayMs;
+    const key = _count++;
     debuglog("setTimeout: scheduling cb", key, "at", runAt,
              "(delay", delayMs, ")");
-    var data = {
+    const data = {
         runAt: runAt,
         func: func,
         params: params,
@@ -85,10 +85,10 @@ module.exports.setTimeout = function(func, delayMs) {
     };
 
     // figure out where it goes in the list
-    var idx = binarySearch(
+    const idx = binarySearch(
         _callbackList, function(el) {
             return el.runAt - runAt;
-        }
+        },
     );
 
     _callbackList.splice(idx, 0, data);
@@ -108,9 +108,9 @@ module.exports.clearTimeout = function(key) {
     }
 
     // remove the element from the list
-    var i;
+    let i;
     for (i = 0; i < _callbackList.length; i++) {
-        var cb = _callbackList[i];
+        const cb = _callbackList[i];
         if (cb.key == key) {
             _callbackList.splice(i, 1);
             break;
@@ -129,29 +129,29 @@ function _scheduleRealCallback() {
         global.clearTimeout(_realCallbackKey);
     }
 
-    var first = _callbackList[0];
+    const first = _callbackList[0];
 
     if (!first) {
         debuglog("_scheduleRealCallback: no more callbacks, not rescheduling");
         return;
     }
 
-    var now = _now();
-    var delayMs = Math.min(first.runAt - now, TIMER_CHECK_PERIOD_MS);
+    const now = _now();
+    const delayMs = Math.min(first.runAt - now, TIMER_CHECK_PERIOD_MS);
 
     debuglog("_scheduleRealCallback: now:", now, "delay:", delayMs);
     _realCallbackKey = global.setTimeout(_runCallbacks, delayMs);
 }
 
 function _runCallbacks() {
-    var cb;
-    var now = _now();
+    let cb;
+    const now = _now();
     debuglog("_runCallbacks: now:", now);
 
     // get the list of things to call
-    var callbacksToRun = [];
+    const callbacksToRun = [];
     while (true) {
-        var first = _callbackList[0];
+        const first = _callbackList[0];
         if (!first || first.runAt > now) {
             break;
         }
@@ -165,10 +165,10 @@ function _runCallbacks() {
     // register their own setTimeouts.
     _scheduleRealCallback();
 
-    for (var i = 0; i < callbacksToRun.length; i++) {
+    for (let i = 0; i < callbacksToRun.length; i++) {
         cb = callbacksToRun[i];
         try {
-            cb.func.apply(null, cb.params);
+            cb.func.apply(global, cb.params);
         } catch (e) {
             console.error("Uncaught exception in callback function",
                           e.stack || e);
@@ -184,12 +184,12 @@ function _runCallbacks() {
  */
 function binarySearch(array, func) {
     // min is inclusive, max exclusive.
-    var min = 0,
+    let min = 0,
         max = array.length;
 
     while (min < max) {
-        var mid = (min + max) >> 1;
-        var res = func(array[mid]);
+        const mid = (min + max) >> 1;
+        const res = func(array[mid]);
         if (res > 0) {
             // the element at 'mid' is too big; set it as the new max.
             max = mid;

src/scheduler.js

@@ -19,10 +19,10 @@ limitations under the License.
  * of requests.
  * @module scheduler
  */
-var utils = require("./utils");
-var q = require("q");
+const utils = require("./utils");
+import Promise from 'bluebird';
 
-var DEBUG = false;  // set true to enable console logging.
+const DEBUG = false;  // set true to enable console logging.
 
 /**
  * Construct a scheduler for Matrix. Requires
@@ -60,7 +60,7 @@ function MatrixScheduler(retryAlgorithm, queueAlgorithm) {
  * @see MatrixScheduler.removeEventFromQueue To remove an event from the queue.
  */
 MatrixScheduler.prototype.getQueueForEvent = function(event) {
-    var name = this.queueAlgorithm(event);
+    const name = this.queueAlgorithm(event);
     if (!name || !this._queues[name]) {
         return null;
     }
@@ -76,13 +76,15 @@ MatrixScheduler.prototype.getQueueForEvent = function(event) {
  * @return {boolean} True if this event was removed.
  */
 MatrixScheduler.prototype.removeEventFromQueue = function(event) {
-    var name = this.queueAlgorithm(event);
+    const name = this.queueAlgorithm(event);
     if (!name || !this._queues[name]) {
         return false;
     }
-    var removed = false;
+    let removed = false;
     utils.removeElement(this._queues[name], function(element) {
         if (element.event.getId() === event.getId()) {
+            // XXX we should probably reject the promise?
+            // https://github.com/matrix-org/matrix-js-sdk/issues/496
             removed = true;
             return true;
         }
@@ -110,7 +112,7 @@ MatrixScheduler.prototype.setProcessFunction = function(fn) {
  * resolved or rejected in due time, else null.
  */
 MatrixScheduler.prototype.queueEvent = function(event) {
-    var queueName = this.queueAlgorithm(event);
+    const queueName = this.queueAlgorithm(event);
     if (!queueName) {
         return null;
     }
@@ -118,15 +120,15 @@ MatrixScheduler.prototype.queueEvent = function(event) {
     if (!this._queues[queueName]) {
         this._queues[queueName] = [];
     }
-    var defer = q.defer();
+    const defer = Promise.defer();
     this._queues[queueName].push({
         event: event,
         defer: defer,
-        attempts: 0
+        attempts: 0,
     });
     debuglog(
         "Queue algorithm dumped event %s into queue '%s'",
-        event.getId(), queueName
+        event.getId(), queueName,
     );
     _startProcessingQueues(this);
     return defer.promise;
@@ -155,7 +157,7 @@ MatrixScheduler.RETRY_BACKOFF_RATELIMIT = function(event, attempts, err) {
     }
 
     if (err.name === "M_LIMIT_EXCEEDED") {
-        var waitTime = err.data.retry_after_ms;
+        const waitTime = err.data.retry_after_ms;
         if (waitTime) {
             return waitTime;
         }
@@ -201,10 +203,10 @@ function _startProcessingQueues(scheduler) {
 
 function _processQueue(scheduler, queueName) {
     // get head of queue
-    var obj = _peekNextEvent(scheduler, queueName);
+    const obj = _peekNextEvent(scheduler, queueName);
     if (!obj) {
         // queue is empty. Mark as inactive and stop recursing.
-        var index = scheduler._activeQueues.indexOf(queueName);
+        const index = scheduler._activeQueues.indexOf(queueName);
         if (index >= 0) {
             scheduler._activeQueues.splice(index, 1);
         }
@@ -213,7 +215,7 @@ function _processQueue(scheduler, queueName) {
     }
     debuglog(
         "Queue '%s' has %s pending events",
-        queueName, scheduler._queues[queueName].length
+        queueName, scheduler._queues[queueName].length,
     );
     // fire the process function and if it resolves, resolve the deferred. Else
     // invoke the retry algorithm.
@@ -227,22 +229,21 @@ function _processQueue(scheduler, queueName) {
     }, function(err) {
         obj.attempts += 1;
         // ask the retry algorithm when/if we should try again
-        var waitTimeMs = scheduler.retryAlgorithm(obj.event, obj.attempts, err);
+        const waitTimeMs = scheduler.retryAlgorithm(obj.event, obj.attempts, err);
         debuglog(
             "retry(%s) err=%s event_id=%s waitTime=%s",
-            obj.attempts, err, obj.event.getId(), waitTimeMs
+            obj.attempts, err, obj.event.getId(), waitTimeMs,
         );
         if (waitTimeMs === -1) {  // give up (you quitter!)
             debuglog(
-                "Queue '%s' giving up on event %s", queueName, obj.event.getId()
+                "Queue '%s' giving up on event %s", queueName, obj.event.getId(),
             );
             // remove this from the queue
             _removeNextEvent(scheduler, queueName);
             obj.defer.reject(err);
             // process next event
             _processQueue(scheduler, queueName);
-        }
-        else {
+        } else {
             setTimeout(function() {
                 _processQueue(scheduler, queueName);
             }, waitTimeMs);
@@ -251,7 +252,7 @@ function _processQueue(scheduler, queueName) {
 }
 
 function _peekNextEvent(scheduler, queueName) {
-    var queue = scheduler._queues[queueName];
+    const queue = scheduler._queues[queueName];
     if (!utils.isArray(queue)) {
         return null;
     }
@@ -259,7 +260,7 @@ function _peekNextEvent(scheduler, queueName) {
 }
 
 function _removeNextEvent(scheduler, queueName) {
-    var queue = scheduler._queues[queueName];
+    const queue = scheduler._queues[queueName];
     if (!utils.isArray(queue)) {
         return null;
     }
@@ -268,7 +269,7 @@ function _removeNextEvent(scheduler, queueName) {
 
 function debuglog() {
     if (DEBUG) {
-        console.log.apply(console, arguments);
+        console.log(...arguments);
     }
 }
 

src/store/indexeddb-local-backend.js

@@ -0,0 +1,571 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+import SyncAccumulator from "../sync-accumulator";
+import utils from "../utils";
+
+const VERSION = 3;
+
+function createDatabase(db) {
+    // Make user store, clobber based on user ID. (userId property of User objects)
+    db.createObjectStore("users", { keyPath: ["userId"] });
+
+    // Make account data store, clobber based on event type.
+    // (event.type property of MatrixEvent objects)
+    db.createObjectStore("accountData", { keyPath: ["type"] });
+
+    // Make /sync store (sync tokens, room data, etc), always clobber (const key).
+    db.createObjectStore("sync", { keyPath: ["clobber"] });
+}
+
+function upgradeSchemaV2(db) {
+    const oobMembersStore = db.createObjectStore(
+        "oob_membership_events", {
+            keyPath: ["room_id", "state_key"],
+        });
+    oobMembersStore.createIndex("room", "room_id");
+}
+
+function upgradeSchemaV3(db) {
+    db.createObjectStore("client_options",
+        { keyPath: ["clobber"]});
+}
+
+
+/**
+ * Helper method to collect results from a Cursor and promiseify it.
+ * @param {ObjectStore|Index} store The store to perform openCursor on.
+ * @param {IDBKeyRange=} keyRange Optional key range to apply on the cursor.
+ * @param {Function} resultMapper A function which is repeatedly called with a
+ * Cursor.
+ * Return the data you want to keep.
+ * @return {Promise<T[]>} Resolves to an array of whatever you returned from
+ * resultMapper.
+ */
+function selectQuery(store, keyRange, resultMapper) {
+    const query = store.openCursor(keyRange);
+    return new Promise((resolve, reject) => {
+        const results = [];
+        query.onerror = (event) => {
+            reject(new Error("Query failed: " + event.target.errorCode));
+        };
+        // collect results
+        query.onsuccess = (event) => {
+            const cursor = event.target.result;
+            if (!cursor) {
+                resolve(results);
+                return; // end of results
+            }
+            results.push(resultMapper(cursor));
+            cursor.continue();
+        };
+    });
+}
+
+function txnAsPromise(txn) {
+    return new Promise((resolve, reject) => {
+        txn.oncomplete = function(event) {
+            resolve(event);
+        };
+        txn.onerror = function(event) {
+            reject(event.target.error);
+        };
+    });
+}
+
+function reqAsEventPromise(req) {
+    return new Promise((resolve, reject) => {
+        req.onsuccess = function(event) {
+            resolve(event);
+        };
+        req.onerror = function(event) {
+            reject(event.target.error);
+        };
+    });
+}
+
+function reqAsPromise(req) {
+    return new Promise((resolve, reject) => {
+        req.onsuccess = () => resolve(req);
+        req.onerror = (err) => reject(err);
+    });
+}
+
+function reqAsCursorPromise(req) {
+    return reqAsEventPromise(req).then((event) => event.target.result);
+}
+
+/**
+ * Does the actual reading from and writing to the indexeddb
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {Object} indexedDBInterface The Indexed DB interface e.g
+ * <code>window.indexedDB</code>
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ */
+const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
+    indexedDBInterface, dbName,
+) {
+    this.indexedDB = indexedDBInterface;
+    this._dbName = "matrix-js-sdk:" + (dbName || "default");
+    this.db = null;
+    this._disconnected = true;
+    this._syncAccumulator = new SyncAccumulator();
+    this._isNewlyCreated = false;
+};
+
+
+LocalIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        if (!this._disconnected) {
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: already connected or connecting`,
+            );
+            return Promise.resolve();
+        }
+
+        this._disconnected = false;
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: connecting...`,
+        );
+        const req = this.indexedDB.open(this._dbName, VERSION);
+        req.onupgradeneeded = (ev) => {
+            const db = ev.target.result;
+            const oldVersion = ev.oldVersion;
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: upgrading from ${oldVersion}`,
+            );
+            if (oldVersion < 1) { // The database did not previously exist.
+                this._isNewlyCreated = true;
+                createDatabase(db);
+            }
+            if (oldVersion < 2) {
+                upgradeSchemaV2(db);
+            }
+            if (oldVersion < 3) {
+                upgradeSchemaV3(db);
+            }
+            // Expand as needed.
+        };
+
+        req.onblocked = () => {
+            console.log(
+                `can't yet open LocalIndexedDBStoreBackend because it is open elsewhere`,
+            );
+        };
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: awaiting connection...`,
+        );
+        return reqAsEventPromise(req).then((ev) => {
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: connected`,
+            );
+            this.db = ev.target.result;
+
+            // add a poorly-named listener for when deleteDatabase is called
+            // so we can close our db connections.
+            this.db.onversionchange = () => {
+                this.db.close();
+            };
+
+            return this._init();
+        });
+    },
+    /** @return {bool} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(this._isNewlyCreated);
+    },
+
+    /**
+     * Having connected, load initial data from the database and prepare for use
+     * @return {Promise} Resolves on success
+     */
+    _init: function() {
+        return Promise.all([
+            this._loadAccountData(),
+            this._loadSyncData(),
+        ]).then(([accountData, syncData]) => {
+            console.log(
+                `LocalIndexedDBStoreBackend: loaded initial data`,
+            );
+            this._syncAccumulator.accumulate({
+                next_batch: syncData.nextBatch,
+                rooms: syncData.roomsData,
+                groups: syncData.groupsData,
+                account_data: {
+                    events: accountData,
+                },
+            });
+        });
+    },
+
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {Promise<event[]>} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return new Promise((resolve, reject) =>{
+            const tx = this.db.transaction(["oob_membership_events"], "readonly");
+            const store = tx.objectStore("oob_membership_events");
+            const roomIndex = store.index("room");
+            const range = IDBKeyRange.only(roomId);
+            const request = roomIndex.openCursor(range);
+
+            const membershipEvents = [];
+            // did we encounter the oob_written marker object
+            // amongst the results? That means OOB member
+            // loading already happened for this room
+            // but there were no members to persist as they
+            // were all known already
+            let oobWritten = false;
+
+            request.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (!cursor) {
+                    // Unknown room
+                    if (!membershipEvents.length && !oobWritten) {
+                        return resolve(null);
+                    }
+                    return resolve(membershipEvents);
+                }
+                const record = cursor.value;
+                if (record.oob_written) {
+                    oobWritten = true;
+                } else {
+                    membershipEvents.push(record);
+                }
+                cursor.continue();
+            };
+            request.onerror = (err) => {
+                reject(err);
+            };
+        }).then((events) => {
+            console.log(`LL: got ${events && events.length}` +
+                ` membershipEvents from storage for room ${roomId} ...`);
+            return events;
+        });
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     */
+    setOutOfBandMembers: async function(roomId, membershipEvents) {
+        console.log(`LL: backend about to store ${membershipEvents.length}` +
+            ` members for ${roomId}`);
+        const tx = this.db.transaction(["oob_membership_events"], "readwrite");
+        const store = tx.objectStore("oob_membership_events");
+        membershipEvents.forEach((e) => {
+            store.put(e);
+        });
+        // aside from all the events, we also write a marker object to the store
+        // to mark the fact that OOB members have been written for this room.
+        // It's possible that 0 members need to be written as all where previously know
+        // but we still need to know whether to return null or [] from getOutOfBandMembers
+        // where null means out of band members haven't been stored yet for this room
+        const markerObject = {
+            room_id: roomId,
+            oob_written: true,
+            state_key: 0,
+        };
+        store.put(markerObject);
+        await txnAsPromise(tx);
+        console.log(`LL: backend done storing for ${roomId}!`);
+    },
+
+    clearOutOfBandMembers: async function(roomId) {
+        // the approach to delete all members for a room
+        // is to get the min and max state key from the index
+        // for that room, and then delete between those
+        // keys in the store.
+        // this should be way faster than deleting every member
+        // individually for a large room.
+        const readTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readonly");
+        const store = readTx.objectStore("oob_membership_events");
+        const roomIndex = store.index("room");
+        const roomRange = IDBKeyRange.only(roomId);
+
+        const minStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "next"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const maxStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "prev"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const [minStateKey, maxStateKey] = await Promise.all(
+            [minStateKeyProm, maxStateKeyProm]);
+
+        const writeTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readwrite");
+        const writeStore = writeTx.objectStore("oob_membership_events");
+        const membersKeyRange = IDBKeyRange.bound(
+            [roomId, minStateKey],
+            [roomId, maxStateKey],
+        );
+
+        console.log(`LL: Deleting all users + marker in storage for ` +
+            `room ${roomId}, with key range:`,
+            [roomId, minStateKey], [roomId, maxStateKey]);
+        await reqAsPromise(writeStore.delete(membersKeyRange));
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        return new Promise((resolve, reject) => {
+            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this.indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                console.log(
+                    `can't yet delete indexeddb ${this._dbName}` +
+                    ` because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                // in firefox, with indexedDB disabled, this fails with a
+                // DOMError. We treat this as non-fatal, so that we can still
+                // use the app.
+                console.warn(
+                    `unable to delete js-sdk store indexeddb: ${ev.target.error}`,
+                );
+                resolve();
+            };
+
+            req.onsuccess = () => {
+                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        });
+    },
+
+    /**
+     * @param {boolean=} copy If false, the data returned is from internal
+     * buffers and must not be mutated. Otherwise, a copy is made before
+     * returning such that the data can be safely mutated. Default: true.
+     *
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function(copy) {
+        if (copy === undefined) copy = true;
+
+        const data = this._syncAccumulator.getJSON();
+        if (!data.nextBatch) return Promise.resolve(null);
+        if (copy) {
+            // We must deep copy the stored data so that the /sync processing code doesn't
+            // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
+            return Promise.resolve(utils.deepCopy(data));
+        } else {
+            return Promise.resolve(data);
+        }
+    },
+
+    getNextBatchToken: function() {
+        return Promise.resolve(this._syncAccumulator.getNextBatchToken());
+    },
+
+    setSyncData: function(syncData) {
+        return Promise.resolve().then(() => {
+            this._syncAccumulator.accumulate(syncData);
+        });
+    },
+
+    syncToDatabase: function(userTuples) {
+        const syncData = this._syncAccumulator.getJSON();
+
+        return Promise.all([
+            this._persistUserPresenceEvents(userTuples),
+            this._persistAccountData(syncData.accountData),
+            this._persistSyncData(
+                syncData.nextBatch, syncData.roomsData, syncData.groupsData,
+            ),
+        ]);
+    },
+
+    /**
+     * Persist rooms /sync data along with the next batch token.
+     * @param {string} nextBatch The next_batch /sync value.
+     * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @param {Object} groupsData The 'groups' /sync data from a SyncAccumulator
+     * @return {Promise} Resolves if the data was persisted.
+     */
+    _persistSyncData: function(nextBatch, roomsData, groupsData) {
+        console.log("Persisting sync data up to ", nextBatch);
+        return Promise.try(() => {
+            const txn = this.db.transaction(["sync"], "readwrite");
+            const store = txn.objectStore("sync");
+            store.put({
+                clobber: "-", // constant key so will always clobber
+                nextBatch: nextBatch,
+                roomsData: roomsData,
+                groupsData: groupsData,
+            }); // put == UPSERT
+            return txnAsPromise(txn);
+        });
+    },
+
+    /**
+     * Persist a list of account data events. Events with the same 'type' will
+     * be replaced.
+     * @param {Object[]} accountData An array of raw user-scoped account data events
+     * @return {Promise} Resolves if the events were persisted.
+     */
+    _persistAccountData: function(accountData) {
+        return Promise.try(() => {
+            const txn = this.db.transaction(["accountData"], "readwrite");
+            const store = txn.objectStore("accountData");
+            for (let i = 0; i < accountData.length; i++) {
+                store.put(accountData[i]); // put == UPSERT
+            }
+            return txnAsPromise(txn);
+        });
+    },
+
+    /**
+     * Persist a list of [user id, presence event] they are for.
+     * Users with the same 'userId' will be replaced.
+     * Presence events should be the event in its raw form (not the Event
+     * object)
+     * @param {Object[]} tuples An array of [userid, event] tuples
+     * @return {Promise} Resolves if the users were persisted.
+     */
+    _persistUserPresenceEvents: function(tuples) {
+        return Promise.try(() => {
+            const txn = this.db.transaction(["users"], "readwrite");
+            const store = txn.objectStore("users");
+            for (const tuple of tuples) {
+                store.put({
+                    userId: tuple[0],
+                    event: tuple[1],
+                }); // put == UPSERT
+            }
+            return txnAsPromise(txn);
+        });
+    },
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * FIXME: It would probably be more sensible to store the events in the
+     * sync.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return Promise.try(() => {
+            const txn = this.db.transaction(["users"], "readonly");
+            const store = txn.objectStore("users");
+            return selectQuery(store, undefined, (cursor) => {
+                return [cursor.value.userId, cursor.value.event];
+            });
+        });
+    },
+
+    /**
+     * Load all the account data events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of raw global account events.
+     */
+    _loadAccountData: function() {
+        console.log(
+            `LocalIndexedDBStoreBackend: loading account data...`,
+        );
+        return Promise.try(() => {
+            const txn = this.db.transaction(["accountData"], "readonly");
+            const store = txn.objectStore("accountData");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            }).then((result) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded account data`,
+                );
+                return result;
+            });
+        });
+    },
+
+    /**
+     * Load the sync data from the database.
+     * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
+     */
+    _loadSyncData: function() {
+        console.log(
+            `LocalIndexedDBStoreBackend: loading sync data...`,
+        );
+        return Promise.try(() => {
+            const txn = this.db.transaction(["sync"], "readonly");
+            const store = txn.objectStore("sync");
+            return selectQuery(store, undefined, (cursor) => {
+                return cursor.value;
+            }).then((results) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded sync data`,
+                );
+                if (results.length > 1) {
+                    console.warn("loadSyncData: More than 1 sync row found.");
+                }
+                return (results.length > 0 ? results[0] : {});
+            });
+        });
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve().then(() => {
+            const txn = this.db.transaction(["client_options"], "readonly");
+            const store = txn.objectStore("client_options");
+            return selectQuery(store, undefined, (cursor) => {
+                if (cursor.value && cursor.value && cursor.value.options) {
+                    return cursor.value.options;
+                }
+            }).then((results) => results[0]);
+        });
+    },
+
+    storeClientOptions: async function(options) {
+        const txn = this.db.transaction(["client_options"], "readwrite");
+        const store = txn.objectStore("client_options");
+        store.put({
+            clobber: "-", // constant key so will always clobber
+            options: options,
+        }); // put == UPSERT
+        await txnAsPromise(txn);
+    },
+};
+
+export default LocalIndexedDBStoreBackend;

src/store/indexeddb-remote-backend.js

@@ -0,0 +1,197 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+/**
+ * An IndexedDB store backend where the actual backend sits in a web
+ * worker.
+ *
+ * Construct a new Indexed Database store backend. This requires a call to
+ * <code>connect()</code> before this store can be used.
+ * @constructor
+ * @param {string} workerScript URL to the worker script
+ * @param {string=} dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {Object} workerApi The web worker compatible interface object
+ */
+const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
+    workerScript, dbName, workerApi,
+) {
+    this._workerScript = workerScript;
+    this._dbName = dbName;
+    this._workerApi = workerApi;
+    this._worker = null;
+    this._nextSeq = 0;
+    // The currently in-flight requests to the actual backend
+    this._inFlight = {
+        // seq: promise,
+    };
+    // Once we start connecting, we keep the promise and re-use it
+    // if we try to connect again
+    this._startPromise = null;
+};
+
+
+RemoteIndexedDBStoreBackend.prototype = {
+    /**
+     * Attempt to connect to the database. This can fail if the user does not
+     * grant permission.
+     * @return {Promise} Resolves if successfully connected.
+     */
+    connect: function() {
+        return this._ensureStarted().then(() => this._doCmd('connect'));
+    },
+
+    /**
+     * Clear the entire database. This should be used when logging out of a client
+     * to prevent mixing data between accounts.
+     * @return {Promise} Resolved when the database is cleared.
+     */
+    clearDatabase: function() {
+        return this._ensureStarted().then(() => this._doCmd('clearDatabase'));
+    },
+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return this._doCmd('isNewlyCreated');
+    },
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return this._doCmd('getSavedSync');
+    },
+
+    getNextBatchToken: function() {
+        return this._doCmd('getNextBatchToken');
+    },
+
+    setSyncData: function(syncData) {
+        return this._doCmd('setSyncData', [syncData]);
+    },
+
+    syncToDatabase: function(users) {
+        return this._doCmd('syncToDatabase', [users]);
+    },
+
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return this._doCmd('getOutOfBandMembers', [roomId]);
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        return this._doCmd('setOutOfBandMembers', [roomId, membershipEvents]);
+    },
+
+    clearOutOfBandMembers: function(roomId) {
+        return this._doCmd('clearOutOfBandMembers', [roomId]);
+    },
+
+    getClientOptions: function() {
+        return this._doCmd('getClientOptions');
+    },
+
+    storeClientOptions: function(options) {
+        return this._doCmd('storeClientOptions', [options]);
+    },
+
+    /**
+     * Load all user presence events from the database. This is not cached.
+     * @return {Promise<Object[]>} A list of presence events in their raw form.
+     */
+    getUserPresenceEvents: function() {
+        return this._doCmd('getUserPresenceEvents');
+    },
+
+    _ensureStarted: function() {
+        if (this._startPromise === null) {
+            this._worker = new this._workerApi(this._workerScript);
+            this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+            // tell the worker the db name.
+            this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
+                console.log("IndexedDB worker is ready");
+            });
+        }
+        return this._startPromise;
+    },
+
+    _doCmd: function(cmd, args) {
+        // wrap in a q so if the postMessage throws,
+        // the promise automatically gets rejected
+        return Promise.resolve().then(() => {
+            const seq = this._nextSeq++;
+            const def = Promise.defer();
+
+            this._inFlight[seq] = def;
+
+            this._worker.postMessage({
+                command: cmd,
+                seq: seq,
+                args: args,
+            });
+
+            return def.promise;
+        });
+    },
+
+    _onWorkerMessage: function(ev) {
+        const msg = ev.data;
+
+        if (msg.command == 'cmd_success' || msg.command == 'cmd_fail') {
+            if (msg.seq === undefined) {
+                console.error("Got reply from worker with no seq");
+                return;
+            }
+
+            const def = this._inFlight[msg.seq];
+            if (def === undefined) {
+                console.error("Got reply for unknown seq " + msg.seq);
+                return;
+            }
+            delete this._inFlight[msg.seq];
+
+            if (msg.command == 'cmd_success') {
+                def.resolve(msg.result);
+            } else {
+                const error = new Error(msg.error.message);
+                error.name = msg.error.name;
+                def.reject(error);
+            }
+        } else {
+            console.warn("Unrecognised message from worker: " + msg);
+        }
+    },
+};
+
+export default RemoteIndexedDBStoreBackend;

src/store/indexeddb-store-worker.js

@@ -0,0 +1,147 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+
+/**
+ * This class lives in the webworker and drives a LocalIndexedDBStoreBackend
+ * controlled by messages from the main process.
+ *
+ * It should be instantiated by a web worker script provided by the application
+ * in a script, for example:
+ *
+ * import {IndexedDBStoreWorker} from 'matrix-js-sdk/lib/indexeddb-worker.js';
+ * const remoteWorker = new IndexedDBStoreWorker(postMessage);
+ * onmessage = remoteWorker.onMessage;
+ *
+ * Note that it is advisable to import this class by referencing the file directly to
+ * avoid a dependency on the whole js-sdk.
+ *
+ */
+class IndexedDBStoreWorker {
+    /**
+     * @param {function} postMessage The web worker postMessage function that
+     * should be used to communicate back to the main script.
+     */
+    constructor(postMessage) {
+        this.backend = null;
+        this.postMessage = postMessage;
+
+        this.onMessage = this.onMessage.bind(this);
+    }
+
+    /**
+     * Passes a message event from the main script into the class. This method
+     * can be directly assigned to the web worker `onmessage` variable.
+     *
+     * @param {Object} ev The message event
+     */
+    onMessage(ev) {
+        const msg = ev.data;
+        let prom;
+
+        switch (msg.command) {
+            case '_setupWorker':
+                this.backend = new LocalIndexedDBStoreBackend(
+                    // this is the 'indexedDB' global (where global != window
+                    // because it's a web worker and there is no window).
+                    indexedDB, msg.args[0],
+                );
+                prom = Promise.resolve();
+                break;
+            case 'connect':
+                prom = this.backend.connect();
+                break;
+            case 'isNewlyCreated':
+                prom = this.backend.isNewlyCreated();
+                break;
+            case 'clearDatabase':
+                prom = this.backend.clearDatabase().then((result) => {
+                    // This returns special classes which can't be cloned
+                    // across to the main script, so don't try.
+                    return {};
+                });
+                break;
+            case 'getSavedSync':
+                prom = this.backend.getSavedSync(false);
+                break;
+            case 'setSyncData':
+                prom = this.backend.setSyncData(...msg.args);
+                break;
+            case 'syncToDatabase':
+                prom = this.backend.syncToDatabase(...msg.args).then(() => {
+                    // This also returns IndexedDB events which are not cloneable
+                    return {};
+                });
+                break;
+            case 'getUserPresenceEvents':
+                prom = this.backend.getUserPresenceEvents();
+                break;
+            case 'getNextBatchToken':
+                prom = this.backend.getNextBatchToken();
+                break;
+            case 'getOutOfBandMembers':
+                prom = this.backend.getOutOfBandMembers(msg.args[0]);
+                break;
+            case 'clearOutOfBandMembers':
+                prom = this.backend.clearOutOfBandMembers(msg.args[0]);
+                break;
+            case 'setOutOfBandMembers':
+                prom = this.backend.setOutOfBandMembers(msg.args[0], msg.args[1]);
+                break;
+            case 'getClientOptions':
+                prom = this.backend.getClientOptions();
+                break;
+            case 'storeClientOptions':
+                prom = this.backend.storeClientOptions(msg.args[0]);
+                break;
+        }
+
+        if (prom === undefined) {
+            this.postMessage({
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Can't be an Error because they're not structured cloneable
+                error: "Unrecognised command",
+            });
+            return;
+        }
+
+        prom.done((ret) => {
+            this.postMessage.call(null, {
+                command: 'cmd_success',
+                seq: msg.seq,
+                result: ret,
+            });
+        }, (err) => {
+            console.error("Error running command: "+msg.command);
+            console.error(err);
+            this.postMessage.call(null, {
+                command: 'cmd_fail',
+                seq: msg.seq,
+                // Just send a string because Error objects aren't cloneable
+                error: {
+                    message: err.message,
+                    name: err.name,
+                },
+            });
+        });
+    }
+}
+
+module.exports = IndexedDBStoreWorker;

src/store/indexeddb.js

@@ -0,0 +1,262 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+import {MatrixInMemoryStore} from "./memory";
+import utils from "../utils";
+import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
+import RemoteIndexedDBStoreBackend from "./indexeddb-remote-backend.js";
+import User from "../models/user";
+import {MatrixEvent} from "../models/event";
+
+/**
+ * This is an internal module. See {@link IndexedDBStore} for the public class.
+ * @module store/indexeddb
+ */
+
+// If this value is too small we'll be writing very often which will cause
+// noticable stop-the-world pauses. If this value is too big we'll be writing
+// so infrequently that the /sync size gets bigger on reload. Writing more
+// often does not affect the length of the pause since the entire /sync
+// response is persisted each time.
+const WRITE_DELAY_MS = 1000 * 60 * 5; // once every 5 minutes
+
+
+/**
+ * Construct a new Indexed Database store, which extends MatrixInMemoryStore.
+ *
+ * This store functions like a MatrixInMemoryStore except it periodically persists
+ * the contents of the store to an IndexedDB backend.
+ *
+ * All data is still kept in-memory but can be loaded from disk by calling
+ * <code>startup()</code>. This can make startup times quicker as a complete
+ * sync from the server is not required. This does not reduce memory usage as all
+ * the data is eagerly fetched when <code>startup()</code> is called.
+ * <pre>
+ * let opts = { localStorage: window.localStorage };
+ * let store = new IndexedDBStore();
+ * await store.startup(); // load from indexed db
+ * let client = sdk.createClient({
+ *     store: store,
+ * });
+ * client.startClient();
+ * client.on("sync", function(state, prevState, data) {
+ *     if (state === "PREPARED") {
+ *         console.log("Started up, now with go faster stripes!");
+ *     }
+ * });
+ * </pre>
+ *
+ * @constructor
+ * @extends MatrixInMemoryStore
+ * @param {Object} opts Options object.
+ * @param {Object} opts.indexedDB The Indexed DB interface e.g.
+ * <code>window.indexedDB</code>
+ * @param {string=} opts.dbName Optional database name. The same name must be used
+ * to open the same database.
+ * @param {string=} opts.workerScript Optional URL to a script to invoke a web
+ * worker with to run IndexedDB queries on the web worker. The IndexedDbStoreWorker
+ * class is provided for this purpose and requires the application to provide a
+ * trivial wrapper script around it.
+ * @param {Object=} opts.workerApi The webWorker API object. If omitted, the global Worker
+ * object will be used if it exists.
+ * @prop {IndexedDBStoreBackend} backend The backend instance. Call through to
+ * this API if you need to perform specific indexeddb actions like deleting the
+ * database.
+ */
+const IndexedDBStore = function IndexedDBStore(opts) {
+    MatrixInMemoryStore.call(this, opts);
+
+    if (!opts.indexedDB) {
+        throw new Error('Missing required option: indexedDB');
+    }
+
+    if (opts.workerScript) {
+        // try & find a webworker-compatible API
+        let workerApi = opts.workerApi;
+        if (!workerApi) {
+            // default to the global Worker object (which is where it in a browser)
+            workerApi = global.Worker;
+        }
+        this.backend = new RemoteIndexedDBStoreBackend(
+            opts.workerScript, opts.dbName, workerApi,
+        );
+    } else {
+        this.backend = new LocalIndexedDBStoreBackend(opts.indexedDB, opts.dbName);
+    }
+
+    this.startedUp = false;
+    this._syncTs = 0;
+
+    // Records the last-modified-time of each user at the last point we saved
+    // the database, such that we can derive the set if users that have been
+    // modified since we last saved.
+    this._userModifiedMap = {
+        // user_id : timestamp
+    };
+};
+utils.inherits(IndexedDBStore, MatrixInMemoryStore);
+
+/**
+ * @return {Promise} Resolved when loaded from indexed db.
+  */
+IndexedDBStore.prototype.startup = function() {
+    if (this.startedUp) {
+        console.log(`IndexedDBStore.startup: already started`);
+        return Promise.resolve();
+    }
+
+    console.log(`IndexedDBStore.startup: connecting to backend`);
+    return this.backend.connect().then(() => {
+        console.log(`IndexedDBStore.startup: loading presence events`);
+        return this.backend.getUserPresenceEvents();
+    }).then((userPresenceEvents) => {
+        console.log(`IndexedDBStore.startup: processing presence events`);
+        userPresenceEvents.forEach(([userId, rawEvent]) => {
+            const u = new User(userId);
+            if (rawEvent) {
+                u.setPresenceEvent(new MatrixEvent(rawEvent));
+            }
+            this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+            this.storeUser(u);
+        });
+    });
+};
+
+/**
+ * @return {Promise} Resolves with a sync response to restore the
+ * client state to where it was at the last save, or null if there
+ * is no saved sync data.
+ */
+IndexedDBStore.prototype.getSavedSync = function() {
+    return this.backend.getSavedSync();
+};
+
+/** @return {Promise<bool>} whether or not the database was newly created in this session. */
+IndexedDBStore.prototype.isNewlyCreated = function() {
+    return this.backend.isNewlyCreated();
+};
+
+/**
+ * @return {Promise} If there is a saved sync, the nextBatch token
+ * for this sync, otherwise null.
+ */
+IndexedDBStore.prototype.getSavedSyncToken = function() {
+    return this.backend.getNextBatchToken();
+},
+
+/**
+ * Delete all data from this store.
+ * @return {Promise} Resolves if the data was deleted from the database.
+ */
+IndexedDBStore.prototype.deleteAllData = function() {
+    MatrixInMemoryStore.prototype.deleteAllData.call(this);
+    return this.backend.clearDatabase().then(() => {
+        console.log("Deleted indexeddb data.");
+    }, (err) => {
+        console.error(`Failed to delete indexeddb data: ${err}`);
+        throw err;
+    });
+};
+
+/**
+ * Whether this store would like to save its data
+ * Note that obviously whether the store wants to save or
+ * not could change between calling this function and calling
+ * save().
+ *
+ * @return {boolean} True if calling save() will actually save
+ *     (at the time this function is called).
+ */
+IndexedDBStore.prototype.wantsSave = function() {
+    const now = Date.now();
+    return now - this._syncTs > WRITE_DELAY_MS;
+};
+
+/**
+ * Possibly write data to the database.
+ * @return {Promise} Promise resolves after the write completes
+ *     (or immediately if no write is performed)
+ */
+IndexedDBStore.prototype.save = function() {
+    if (this.wantsSave()) {
+        return this._reallySave();
+    }
+    return Promise.resolve();
+};
+
+IndexedDBStore.prototype._reallySave = function() {
+    this._syncTs = Date.now(); // set now to guard against multi-writes
+
+    // work out changed users (this doesn't handle deletions but you
+    // can't 'delete' users as they are just presence events).
+    const userTuples = [];
+    for (const u of this.getUsers()) {
+        if (this._userModifiedMap[u.userId] === u.getLastModifiedTime()) continue;
+        if (!u.events.presence) continue;
+
+        userTuples.push([u.userId, u.events.presence.event]);
+
+        // note that we've saved this version of the user
+        this._userModifiedMap[u.userId] = u.getLastModifiedTime();
+    }
+
+    return this.backend.syncToDatabase(userTuples).catch((err) => {
+        console.error("sync fail:", err);
+    });
+};
+
+IndexedDBStore.prototype.setSyncData = function(syncData) {
+    return this.backend.setSyncData(syncData);
+};
+
+/**
+ * Returns the out-of-band membership events for this room that
+ * were previously loaded.
+ * @param {string} roomId
+ * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+ * @returns {null} in case the members for this room haven't been stored yet
+ */
+IndexedDBStore.prototype.getOutOfBandMembers = function(roomId) {
+    return this.backend.getOutOfBandMembers(roomId);
+};
+
+/**
+ * Stores the out-of-band membership events for this room. Note that
+ * it still makes sense to store an empty array as the OOB status for the room is
+ * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+ * @param {string} roomId
+ * @param {event[]} membershipEvents the membership events to store
+ * @returns {Promise} when all members have been stored
+ */
+IndexedDBStore.prototype.setOutOfBandMembers = function(roomId, membershipEvents) {
+    return this.backend.setOutOfBandMembers(roomId, membershipEvents);
+};
+
+IndexedDBStore.prototype.clearOutOfBandMembers = function(roomId) {
+    return this.backend.clearOutOfBandMembers(roomId);
+};
+
+IndexedDBStore.prototype.getClientOptions = function() {
+    return this.backend.getClientOptions();
+};
+
+IndexedDBStore.prototype.storeClientOptions = function(options) {
+    return this.backend.storeClientOptions(options);
+};
+
+module.exports.IndexedDBStore = IndexedDBStore;

src/store/memory.js

@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,22 +20,25 @@ limitations under the License.
  * This is an internal module. See {@link MatrixInMemoryStore} for the public class.
  * @module store/memory
  */
- var utils = require("../utils");
- var User = require("../models/user");
+const utils = require("../utils");
+const User = require("../models/user");
+import Promise from 'bluebird';
 
 /**
  * Construct a new in-memory data store for the Matrix Client.
  * @constructor
  * @param {Object=} opts Config options
  * @param {LocalStorage} opts.localStorage The local storage instance to persist
- * some forms of data such as tokens. Rooms will NOT be stored. See
- * {@link WebStorageStore} to persist rooms.
+ * some forms of data such as tokens. Rooms will NOT be stored.
  */
 module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
     opts = opts || {};
     this.rooms = {
         // roomId: Room
     };
+    this.groups = {
+        // groupId: Group
+    };
     this.users = {
         // userId: User
     };
@@ -47,6 +52,10 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
         // type : content
     };
     this.localStorage = opts.localStorage;
+    this._oobMembers = {
+        // roomId: [member events]
+    };
+    this._clientOptions = {};
 };
 
 module.exports.MatrixInMemoryStore.prototype = {
@@ -59,6 +68,10 @@ module.exports.MatrixInMemoryStore.prototype = {
         return this.syncToken;
     },
 
+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },
 
     /**
      * Set the token to stream from.
@@ -68,6 +81,31 @@ module.exports.MatrixInMemoryStore.prototype = {
         this.syncToken = token;
     },
 
+    /**
+     * Store the given room.
+     * @param {Group} group The group to be stored
+     */
+    storeGroup: function(group) {
+        this.groups[group.groupId] = group;
+    },
+
+    /**
+     * Retrieve a group by its group ID.
+     * @param {string} groupId The group ID.
+     * @return {Group} The group or null.
+     */
+    getGroup: function(groupId) {
+        return this.groups[groupId] || null;
+    },
+
+    /**
+     * Retrieve all known groups.
+     * @return {Group[]} A list of groups, which may be empty.
+     */
+    getGroups: function() {
+        return utils.values(this.groups);
+    },
+
     /**
      * Store the given room.
      * @param {Room} room The room to be stored. All properties must be stored.
@@ -78,7 +116,7 @@ module.exports.MatrixInMemoryStore.prototype = {
         // map up-to-date.
         room.currentState.on("RoomState.members", this._onRoomMember.bind(this));
         // add existing members
-        var self = this;
+        const self = this;
         room.currentState.getMembers().forEach(function(m) {
             self._onRoomMember(null, room.currentState, m);
         });
@@ -98,12 +136,12 @@ module.exports.MatrixInMemoryStore.prototype = {
             return;
         }
 
-        var user = this.users[member.userId] || new User(member.userId);
+        const user = this.users[member.userId] || new User(member.userId);
         if (member.name) {
             user.setDisplayName(member.name);
             if (member.events.member) {
                 user.setRawDisplayName(
-                    member.events.member.getDirectionalContent().displayname
+                    member.events.member.getDirectionalContent().displayname,
                 );
             }
         }
@@ -203,7 +241,9 @@ module.exports.MatrixInMemoryStore.prototype = {
      * @param {Filter} filter
      */
     storeFilter: function(filter) {
-        if (!filter) { return; }
+        if (!filter) {
+            return;
+        }
         if (!this.filters[filter.userId]) {
             this.filters[filter.userId] = {};
         }
@@ -234,8 +274,7 @@ module.exports.MatrixInMemoryStore.prototype = {
         }
         try {
             return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
-        }
-        catch (e) {}
+        } catch (e) {}
         return null;
     },
 
@@ -250,8 +289,7 @@ module.exports.MatrixInMemoryStore.prototype = {
         }
         try {
             this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
-        }
-        catch (e) {}
+        } catch (e) {}
     },
 
     /**
@@ -261,7 +299,7 @@ module.exports.MatrixInMemoryStore.prototype = {
      * @param {Array<MatrixEvent>} events The events to store.
      */
     storeAccountDataEvents: function(events) {
-        var self = this;
+        const self = this;
         events.forEach(function(event) {
             self.accountData[event.getType()] = event;
         });
@@ -276,9 +314,106 @@ module.exports.MatrixInMemoryStore.prototype = {
         return this.accountData[eventType];
     },
 
-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return Promise.resolve();
+    },
 
-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing as this store doesn't require starting up.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return Promise.resolve();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return Promise.resolve(null);
+    },
+
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
+    /**
+     * Delete all data from this store.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        this.rooms = {
+            // roomId: Room
+        };
+        this.users = {
+            // userId: User
+        };
+        this.syncToken = null;
+        this.filters = {
+            // userId: {
+            //    filterId: Filter
+            // }
+        };
+        this.accountData = {
+            // type : content
+        };
+        return Promise.resolve();
+    },
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return Promise.resolve(this._oobMembers[roomId] || null);
+    },
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        this._oobMembers[roomId] = membershipEvents;
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve(this._clientOptions);
+    },
+
+    storeClientOptions: function(options) {
+        this._clientOptions = Object.assign({}, options);
+        return Promise.resolve();
+    },
 };

src/store/session/webstorage.js

@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,10 +21,10 @@ limitations under the License.
  * @module store/session/webstorage
  */
 
-var utils = require("../../utils");
+const utils = require("../../utils");
 
-var DEBUG = false;  // set true to enable console logging.
-var E2E_PREFIX = "session.e2e.";
+const DEBUG = false;  // set true to enable console logging.
+const E2E_PREFIX = "session.e2e.";
 
 /**
  * Construct a web storage session store, capable of storing account keys,
@@ -37,25 +39,29 @@ function WebStorageSessionStore(webStore) {
     this.store = webStore;
     if (!utils.isFunction(webStore.getItem) ||
         !utils.isFunction(webStore.setItem) ||
-        !utils.isFunction(webStore.removeItem)) {
+        !utils.isFunction(webStore.removeItem) ||
+        !utils.isFunction(webStore.key) ||
+        typeof(webStore.length) !== 'number'
+       ) {
         throw new Error(
-            "Supplied webStore does not meet the WebStorage API interface"
+            "Supplied webStore does not meet the WebStorage API interface",
         );
     }
 }
 
 WebStorageSessionStore.prototype = {
-
     /**
-     * Store the end to end account for the logged-in user.
-     * @param {string} account Base64 encoded account.
+     * Remove the stored end to end account for the logged-in user.
      */
-    storeEndToEndAccount: function(account) {
-        this.store.setItem(KEY_END_TO_END_ACCOUNT, account);
+    removeEndToEndAccount: function() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
     },
 
     /**
      * Load the end to end account for the logged-in user.
+     * Note that the end-to-end account is now stored in the
+     * crypto store rather than here: this remains here so
+     * old sessions can be migrated out of the session store.
      * @return {?string} Base64 encoded account.
      */
     getEndToEndAccount: function() {
@@ -63,51 +69,40 @@ WebStorageSessionStore.prototype = {
     },
 
     /**
-     * Store a flag indicating that we have announced the new device.
+     * Retrieves the known devices for all users.
+     * @return {object} A map from user ID to map of device ID to keys for the device.
      */
-    setDeviceAnnounced: function() {
-        this.store.setItem(KEY_END_TO_END_ANNOUNCED, "true");
+    getAllEndToEndDevices: function() {
+        const prefix = keyEndToEndDevicesForUser('');
+        const devices = {};
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            const userId = key.substr(prefix.length);
+            if (key.startsWith(prefix)) devices[userId] = getJsonItem(this.store, key);
+        }
+        return devices;
+    },
+
+    getEndToEndDeviceTrackingStatus: function() {
+        return getJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
     },
 
     /**
-     * Check if the "device announced" flag is set
+     * Get the sync token corresponding to the device list.
      *
-     * @return {boolean} true if the "device announced" flag has been set.
+     * @return {String?} token
      */
-    getDeviceAnnounced: function() {
-        return this.store.getItem(KEY_END_TO_END_ANNOUNCED) == "true";
+    getEndToEndDeviceSyncToken: function() {
+        return getJsonItem(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN);
     },
 
     /**
-     * Stores the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @param {object} devices A map from device ID to keys for the device.
+     * Removes all end to end device data from the store
      */
-    storeEndToEndDevicesForUser: function(userId, devices) {
-        setJsonItem(this.store, keyEndToEndDevicesForUser(userId), devices);
-    },
-
-    /**
-     * Retrieves the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @return {object} A map from device ID to keys for the device.
-     */
-    getEndToEndDevicesForUser: function(userId) {
-        return getJsonItem(this.store, keyEndToEndDevicesForUser(userId));
-    },
-
-    /**
-     * Store a session between the logged-in user and another device
-     * @param {string} deviceKey The public key of the other device.
-     * @param {string} sessionId The ID for this end-to-end session.
-     * @param {string} session Base64 encoded end-to-end session.
-     */
-    storeEndToEndSession: function(deviceKey, sessionId, session) {
-        var sessions = this.getEndToEndSessions(deviceKey) || {};
-        sessions[sessionId] = session;
-        setJsonItem(
-            this.store, keyEndToEndSessions(deviceKey), sessions
-        );
+    removeEndToEndDeviceData: function() {
+        removeByPrefix(this.store, keyEndToEndDevicesForUser(''));
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN);
     },
 
     /**
@@ -120,37 +115,86 @@ WebStorageSessionStore.prototype = {
         return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
     },
 
+    /**
+     * Retrieve all end-to-end sessions between the logged-in user and other
+     * devices.
+     * @return {object} A map of {deviceKey -> {sessionId -> session pickle}}
+     */
+    getAllEndToEndSessions: function() {
+        const deviceKeys = getKeysWithPrefix(this.store, keyEndToEndSessions(''));
+        const results = {};
+        for (const k of deviceKeys) {
+            const unprefixedKey = k.substr(keyEndToEndSessions('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
+    },
+
+    /**
+     * Remove all end-to-end sessions from the store
+     * This is used after migrating sessions awat from the sessions store.
+     */
+    removeAllEndToEndSessions: function() {
+        removeByPrefix(this.store, keyEndToEndSessions(''));
+    },
+
+    /**
+     * Retrieve a list of all known inbound group sessions
+     *
+     * @return {{senderKey: string, sessionId: string}}
+     */
+    getAllEndToEndInboundGroupSessionKeys: function() {
+        const prefix = E2E_PREFIX + 'inboundgroupsessions/';
+        const result = [];
+        for (let i = 0; i < this.store.length; i++) {
+            const key = this.store.key(i);
+            if (!key.startsWith(prefix)) {
+                continue;
+            }
+            // we can't use split, as the components we are trying to split out
+            // might themselves contain '/' characters. We rely on the
+            // senderKey being a (32-byte) curve25519 key, base64-encoded
+            // (hence 43 characters long).
+
+            result.push({
+                senderKey: key.substr(prefix.length, 43),
+                sessionId: key.substr(prefix.length + 44),
+            });
+        }
+        return result;
+    },
+
     getEndToEndInboundGroupSession: function(senderKey, sessionId) {
-        var key = keyEndToEndInboundGroupSession(senderKey, sessionId);
+        const key = keyEndToEndInboundGroupSession(senderKey, sessionId);
         return this.store.getItem(key);
     },
 
-    storeEndToEndInboundGroupSession: function(senderKey, sessionId, pickledSession) {
-        var key = keyEndToEndInboundGroupSession(senderKey, sessionId);
-        return this.store.setItem(key, pickledSession);
+    removeAllEndToEndInboundGroupSessions: function() {
+        removeByPrefix(this.store, E2E_PREFIX + 'inboundgroupsessions/');
     },
 
     /**
-     * Store the end-to-end state for a room.
-     * @param {string} roomId The room's ID.
-     * @param {object} roomInfo The end-to-end info for the room.
+     * Get the end-to-end state for all rooms
+     * @return {object} roomId -> object with the end-to-end info for the room.
      */
-    storeEndToEndRoom: function(roomId, roomInfo) {
-        setJsonItem(this.store, keyEndToEndRoom(roomId), roomInfo);
+    getAllEndToEndRooms: function() {
+        const roomKeys = getKeysWithPrefix(this.store, keyEndToEndRoom(''));
+        const results = {};
+        for (const k of roomKeys) {
+            const unprefixedKey = k.substr(keyEndToEndRoom('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
     },
 
-    /**
-     * Get the end-to-end state for a room
-     * @param {string} roomId The room's ID.
-     * @return {object} The end-to-end info for the room.
-     */
-    getEndToEndRoom: function(roomId) {
-        return getJsonItem(this.store, keyEndToEndRoom(roomId));
-    }
+    removeAllEndToEndRooms: function() {
+        removeByPrefix(this.store, keyEndToEndRoom(''));
+    },
 };
 
-var KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
-var KEY_END_TO_END_ANNOUNCED = E2E_PREFIX + "announced";
+const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
+const KEY_END_TO_END_DEVICE_SYNC_TOKEN = E2E_PREFIX + "device_sync_token";
+const KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS = E2E_PREFIX + "device_tracking";
 
 function keyEndToEndDevicesForUser(userId) {
     return E2E_PREFIX + "devices/" + userId;
@@ -170,22 +214,39 @@ function keyEndToEndRoom(roomId) {
 
 function getJsonItem(store, key) {
     try {
+        // if the key is absent, store.getItem() returns null, and
+        // JSON.parse(null) === null, so this returns null.
         return JSON.parse(store.getItem(key));
-    }
-    catch (e) {
+    } catch (e) {
         debuglog("Failed to get key %s: %s", key, e);
         debuglog(e.stack);
     }
     return null;
 }
 
-function setJsonItem(store, key, val) {
-    store.setItem(key, JSON.stringify(val));
+function getKeysWithPrefix(store, prefix) {
+    const results = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) results.push(key);
+    }
+    return results;
+}
+
+function removeByPrefix(store, prefix) {
+    const toRemove = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) toRemove.push(key);
+    }
+    for (const key of toRemove) {
+        store.removeItem(key);
+    }
 }
 
 function debuglog() {
     if (DEBUG) {
-        console.log.apply(console, arguments);
+        console.log(...arguments);
     }
 }
 

src/store/stub.js

@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,6 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 "use strict";
+import Promise from 'bluebird';
 /**
  * This is an internal module.
  * @module store/stub
@@ -29,6 +32,11 @@ function StubStore() {
 
 StubStore.prototype = {
 
+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },
+
     /**
      * Get the sync token.
      * @return {string}
@@ -45,6 +53,30 @@ StubStore.prototype = {
         this.fromToken = token;
     },
 
+    /**
+     * No-op.
+     * @param {Group} group
+     */
+    storeGroup: function(group) {
+    },
+
+    /**
+     * No-op.
+     * @param {string} groupId
+     * @return {null}
+     */
+    getGroup: function(groupId) {
+        return null;
+    },
+
+    /**
+     * No-op.
+     * @return {Array} An empty array.
+     */
+    getGroups: function() {
+        return [];
+    },
+
     /**
      * No-op.
      * @param {Room} room
@@ -180,11 +212,83 @@ StubStore.prototype = {
 
     },
 
-    // TODO
-    //setMaxHistoryPerRoom: function(maxHistory) {},
+    /**
+     * setSyncData does nothing as there is no backing data store.
+     *
+     * @param {Object} syncData The sync data
+     * @return {Promise} An immediately resolved promise.
+     */
+    setSyncData: function(syncData) {
+        return Promise.resolve();
+    },
 
-    // TODO
-    //reapOldMessages: function() {},
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
+    /**
+     * Save does nothing as there is no backing data store.
+     */
+    save: function() {},
+
+    /**
+     * Startup does nothing.
+     * @return {Promise} An immediately resolved promise.
+     */
+    startup: function() {
+        return Promise.resolve();
+    },
+
+    /**
+     * @return {Promise} Resolves with a sync response to restore the
+     * client state to where it was at the last save, or null if there
+     * is no saved sync data.
+     */
+    getSavedSync: function() {
+        return Promise.resolve(null);
+    },
+
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
+    /**
+     * Delete all data from this store. Does nothing since this store
+     * doesn't store anything.
+     * @return {Promise} An immediately resolved promise.
+     */
+    deleteAllData: function() {
+        return Promise.resolve();
+    },
+
+    getOutOfBandMembers: function() {
+        return Promise.resolve(null);
+    },
+
+    setOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    clearOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve();
+    },
+
+    storeClientOptions: function() {
+        return Promise.resolve();
+    },
 };
 
 /** Stub Store class. */