v0.10.1

Stop the error from propagating to the txn error handler and
aborting the rest of the transaction.

.eslintrc.js

@@ -9,6 +9,10 @@ module.exports = {
     env: {
         browser: true,
         node: true,
+
+        // babel's transform-runtime converts references to ES6 globals such as
+        // Promise and Map to core-js polyfills, so we can use ES6 globals.
+        es6: true,
     },
     extends: ["eslint:recommended", "google"],
     rules: {

CHANGELOG.md

@@ -1,3 +1,228 @@
+Changes in [0.10.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.1) (2018-04-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0...v0.10.1)
+
+ * Log IDB errors
+   [\#638](https://github.com/matrix-org/matrix-js-sdk/pull/638)
+ * Ignore inserts of duplicate inbound group sessions
+   [\#639](https://github.com/matrix-org/matrix-js-sdk/pull/639)
+ * Ignore inserts of dup inbound group sessions, pt 2
+   [\#641](https://github.com/matrix-org/matrix-js-sdk/pull/641)
+
+Changes in [0.10.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0) (2018-04-11)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.2...v0.10.0)
+
+ * No changes
+
+Changes in [0.10.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.2) (2018-04-09)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.1...v0.10.0-rc.2)
+
+ * Add wrapper for group join API
+ * Add wrapped API to set group join\_policy
+
+Changes in [0.10.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.1) (2018-03-19)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.2...v0.10.0-rc.1)
+
+ * Fix duplicated state events in timeline from peek
+   [\#630](https://github.com/matrix-org/matrix-js-sdk/pull/630)
+ * Create indexeddb worker when starting the store
+   [\#627](https://github.com/matrix-org/matrix-js-sdk/pull/627)
+ * Fix indexeddb logging
+   [\#626](https://github.com/matrix-org/matrix-js-sdk/pull/626)
+ * Don't do /keys/changes on incremental sync
+   [\#625](https://github.com/matrix-org/matrix-js-sdk/pull/625)
+ * Don't mark devicelist dirty unnecessarily
+   [\#623](https://github.com/matrix-org/matrix-js-sdk/pull/623)
+ * Cache the joined member count for a room state
+   [\#619](https://github.com/matrix-org/matrix-js-sdk/pull/619)
+ * Fix JS doc
+   [\#618](https://github.com/matrix-org/matrix-js-sdk/pull/618)
+ * Precompute push actions for state events
+   [\#617](https://github.com/matrix-org/matrix-js-sdk/pull/617)
+ * Fix bug where global "Never send to unverified..." is ignored
+   [\#616](https://github.com/matrix-org/matrix-js-sdk/pull/616)
+ * Intern legacy top-level 'membership' field
+   [\#615](https://github.com/matrix-org/matrix-js-sdk/pull/615)
+ * Don't synthesize RR for m.room.redaction as causes the RR to go missing.
+   [\#598](https://github.com/matrix-org/matrix-js-sdk/pull/598)
+ * Make Events create Dates on demand
+   [\#613](https://github.com/matrix-org/matrix-js-sdk/pull/613)
+ * Stop cloning events when adding to state
+   [\#612](https://github.com/matrix-org/matrix-js-sdk/pull/612)
+ * De-dup code: use the initialiseState function
+   [\#611](https://github.com/matrix-org/matrix-js-sdk/pull/611)
+ * Create sentinel members on-demand
+   [\#610](https://github.com/matrix-org/matrix-js-sdk/pull/610)
+ * Some more doc on how sentinels work
+   [\#609](https://github.com/matrix-org/matrix-js-sdk/pull/609)
+ * Migrate room encryption store to crypto store
+   [\#597](https://github.com/matrix-org/matrix-js-sdk/pull/597)
+ * add parameter to getIdentityServerUrl to strip the protocol for invites
+   [\#600](https://github.com/matrix-org/matrix-js-sdk/pull/600)
+ * Move Device Tracking Data to Crypto Store
+   [\#594](https://github.com/matrix-org/matrix-js-sdk/pull/594)
+ * Optimise pushprocessor
+   [\#591](https://github.com/matrix-org/matrix-js-sdk/pull/591)
+ * Set event error before emitting
+   [\#592](https://github.com/matrix-org/matrix-js-sdk/pull/592)
+ * Add event type for stickers [WIP]
+   [\#590](https://github.com/matrix-org/matrix-js-sdk/pull/590)
+ * Migrate inbound sessions to cryptostore
+   [\#587](https://github.com/matrix-org/matrix-js-sdk/pull/587)
+ * Disambiguate names if they contain an mxid
+   [\#588](https://github.com/matrix-org/matrix-js-sdk/pull/588)
+ * Check for sessions in indexeddb before migrating
+   [\#585](https://github.com/matrix-org/matrix-js-sdk/pull/585)
+ * Emit an event for crypto store migration
+   [\#586](https://github.com/matrix-org/matrix-js-sdk/pull/586)
+ * Supporting fixes For making UnknownDeviceDialog not pop up automatically
+   [\#575](https://github.com/matrix-org/matrix-js-sdk/pull/575)
+ * Move sessions to the crypto store
+   [\#584](https://github.com/matrix-org/matrix-js-sdk/pull/584)
+ * Change crypto store transaction API
+   [\#582](https://github.com/matrix-org/matrix-js-sdk/pull/582)
+ * Add some missed copyright notices
+   [\#581](https://github.com/matrix-org/matrix-js-sdk/pull/581)
+ * Move Olm account to IndexedDB
+   [\#579](https://github.com/matrix-org/matrix-js-sdk/pull/579)
+ * Fix logging of DecryptionErrors to be more useful
+   [\#580](https://github.com/matrix-org/matrix-js-sdk/pull/580)
+ * [BREAKING] Change the behaviour of the unverfied devices blacklist flag
+   [\#568](https://github.com/matrix-org/matrix-js-sdk/pull/568)
+ * Support set_presence=offline for syncing
+   [\#557](https://github.com/matrix-org/matrix-js-sdk/pull/557)
+ * Consider cases where the sender may not redact their own event
+   [\#556](https://github.com/matrix-org/matrix-js-sdk/pull/556)
+
+Changes in [0.9.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.2) (2017-12-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.1...v0.9.2)
+
+
+Changes in [0.9.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.1) (2017-11-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0...v0.9.1)
+
+ * Fix the force TURN option
+   [\#577](https://github.com/matrix-org/matrix-js-sdk/pull/577)
+
+Changes in [0.9.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0) (2017-11-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0-rc.1...v0.9.0)
+
+
+Changes in [0.9.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0-rc.1) (2017-11-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5...v0.9.0-rc.1)
+
+ * Modify addRoomToGroup to allow setting isPublic, create alias
+   updateGroupRoomAssociation
+   [\#567](https://github.com/matrix-org/matrix-js-sdk/pull/567)
+ * Expose more functionality of pushprocessor
+   [\#565](https://github.com/matrix-org/matrix-js-sdk/pull/565)
+ * Function for working out notif trigger permission
+   [\#566](https://github.com/matrix-org/matrix-js-sdk/pull/566)
+ * keep track of event ID and timestamp of decrypted messages
+   [\#555](https://github.com/matrix-org/matrix-js-sdk/pull/555)
+ * Fix notifEvent computation
+   [\#564](https://github.com/matrix-org/matrix-js-sdk/pull/564)
+ * Fix power level of sentinel members
+   [\#563](https://github.com/matrix-org/matrix-js-sdk/pull/563)
+ * don't try to decrypt a redacted message (fixes vector-im/riot-web#3744)
+   [\#554](https://github.com/matrix-org/matrix-js-sdk/pull/554)
+ * Support room notifs
+   [\#562](https://github.com/matrix-org/matrix-js-sdk/pull/562)
+ * Fix the glob-to-regex code
+   [\#558](https://github.com/matrix-org/matrix-js-sdk/pull/558)
+
+Changes in [0.8.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5) (2017-10-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5-rc.1...v0.8.5)
+
+ * Make unknown pushrule conditions not match
+   [\#559](https://github.com/matrix-org/matrix-js-sdk/pull/559)
+
+Changes in [0.8.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5-rc.1) (2017-10-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.4...v0.8.5-rc.1)
+
+ * Implement wrapper API for removing a room from a group
+   [\#553](https://github.com/matrix-org/matrix-js-sdk/pull/553)
+ * Fix typo which resulted in stuck key download requests
+   [\#552](https://github.com/matrix-org/matrix-js-sdk/pull/552)
+ * Store group when it's created
+   [\#549](https://github.com/matrix-org/matrix-js-sdk/pull/549)
+ * Luke/groups remove rooms users from summary
+   [\#548](https://github.com/matrix-org/matrix-js-sdk/pull/548)
+ * Clean on prepublish
+   [\#546](https://github.com/matrix-org/matrix-js-sdk/pull/546)
+ * Implement wrapper APIs for adding rooms to group summary
+   [\#545](https://github.com/matrix-org/matrix-js-sdk/pull/545)
+
+Changes in [0.8.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.4) (2017-09-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3...v0.8.4)
+
+ * Fix build issue
+
+Changes in [0.8.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3) (2017-09-20)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3-rc.1...v0.8.3)
+
+ * No changes
+
+Changes in [0.8.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3-rc.1) (2017-09-19)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.2...v0.8.3-rc.1)
+
+ * consume trailing slash when creating Matrix Client in HS and IS urls
+   [\#526](https://github.com/matrix-org/matrix-js-sdk/pull/526)
+ * Add ignore users API
+   [\#539](https://github.com/matrix-org/matrix-js-sdk/pull/539)
+ * Upgrade to jsdoc 3.5.5
+   [\#540](https://github.com/matrix-org/matrix-js-sdk/pull/540)
+ * Make re-emitting events much more memory efficient
+   [\#538](https://github.com/matrix-org/matrix-js-sdk/pull/538)
+ * Only re-emit events from Event objects if needed
+   [\#536](https://github.com/matrix-org/matrix-js-sdk/pull/536)
+ * Handle 'left' users in the deviceList mananagement
+   [\#535](https://github.com/matrix-org/matrix-js-sdk/pull/535)
+ * Factor out devicelist integration tests to a separate file
+   [\#534](https://github.com/matrix-org/matrix-js-sdk/pull/534)
+ * Refactor sync._sync as an async function
+   [\#533](https://github.com/matrix-org/matrix-js-sdk/pull/533)
+ * Add es6 to eslint environments
+   [\#532](https://github.com/matrix-org/matrix-js-sdk/pull/532)
+
+Changes in [0.8.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.2) (2017-08-24)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1...v0.8.2)
+
+ * Handle m.call.* events which are decrypted asynchronously
+   [\#530](https://github.com/matrix-org/matrix-js-sdk/pull/530)
+ * Re-emit events from, er, Event objects
+   [\#529](https://github.com/matrix-org/matrix-js-sdk/pull/529)
+
+Changes in [0.8.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1) (2017-08-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1-rc.1...v0.8.1)
+
+ * [No changes]
+
+Changes in [0.8.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1-rc.1) (2017-08-22)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.0...v0.8.1-rc.1)
+
+ * Fix error handling in interactive-auth
+   [\#527](https://github.com/matrix-org/matrix-js-sdk/pull/527)
+ * Make lots of OlmDevice asynchronous
+   [\#524](https://github.com/matrix-org/matrix-js-sdk/pull/524)
+ * Make crypto.decryptMessage return decryption results
+   [\#523](https://github.com/matrix-org/matrix-js-sdk/pull/523)
+
 Changes in [0.8.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.0) (2017-08-15)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.13...v0.8.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "0.8.0",
+  "version": "0.10.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {
@@ -11,11 +11,12 @@
     "check": "npm run test:build && _mocha --recursive specbuild --colors",
     "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
     "start": "babel -s -w -d lib src",
+    "clean": "rimraf lib dist",
     "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
     "dist": "npm run build",
     "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 109 src spec",
-    "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
+    "lint": "eslint --max-warnings 102 src spec",
+    "prepublish": "npm run clean && npm run build && git rev-parse HEAD > git-revision.txt"
   },
   "repository": {
     "url": "https://github.com/matrix-org/matrix-js-sdk"
@@ -49,6 +50,7 @@
   ],
   "dependencies": {
     "another-json": "^0.2.0",
+    "babel-runtime": "^6.26.0",
     "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
     "content-type": "^1.0.2",
@@ -67,7 +69,7 @@
     "exorcist": "^0.4.0",
     "expect": "^1.20.2",
     "istanbul": "^0.4.5",
-    "jsdoc": "^3.5.0",
+    "jsdoc": "^3.5.5",
     "lolex": "^1.5.2",
     "matrix-mock-request": "^1.2.0",
     "mocha": "^3.2.0",

spec/TestClient.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,6 +26,7 @@ import testUtils from './test-utils';
 import MockHttpBackend from 'matrix-mock-request';
 import expect from 'expect';
 import Promise from 'bluebird';
+import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -33,19 +35,32 @@ import Promise from 'bluebird';
  * @param {string} userId
  * @param {string} deviceId
  * @param {string} accessToken
+ *
+ * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
+ *     session store. If undefined, we will create a MockStorageApi.
  */
-export default function TestClient(userId, deviceId, accessToken) {
+export default function TestClient(
+    userId, deviceId, accessToken, sessionStoreBackend,
+) {
     this.userId = userId;
     this.deviceId = deviceId;
 
-    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
+    if (sessionStoreBackend === undefined) {
+        sessionStoreBackend = new testUtils.MockStorageApi();
+    }
+    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
+
+    // expose this so the tests can get to it
+    this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+
     this.httpBackend = new MockHttpBackend();
     this.client = sdk.createClient({
         baseUrl: "http://" + userId + ".test.server",
         userId: userId,
         accessToken: accessToken,
         deviceId: deviceId,
-        sessionStore: this.storage,
+        sessionStore: sessionStore,
+        cryptoStore: this.cryptoStore,
         request: this.httpBackend.requestFn,
     });
 

spec/integ/devicelist-integ-spec.js

@@ -0,0 +1,400 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import TestClient from '../TestClient';
+import testUtils from '../test-utils';
+
+const ROOM_ID = "!room:id";
+
+/**
+ * get a /sync response which contains a single e2e room (ROOM_ID), with the
+ * members given
+ *
+ * @param {string[]} roomMembers
+ *
+ * @return {object} sync response
+ */
+function getSyncResponse(roomMembers) {
+    const stateEvents = [
+        testUtils.mkEvent({
+            type: 'm.room.encryption',
+            skey: '',
+            content: {
+                algorithm: 'm.megolm.v1.aes-sha2',
+            },
+        }),
+    ];
+
+    Array.prototype.push.apply(
+        stateEvents,
+        roomMembers.map(
+            (m) => testUtils.mkMembership({
+                mship: 'join',
+                sender: m,
+            }),
+        ),
+    );
+
+    const syncResponse = {
+        next_batch: 1,
+        rooms: {
+            join: {
+                [ROOM_ID]: {
+                    state: {
+                        events: stateEvents,
+                    },
+                },
+            },
+        },
+    };
+
+    return syncResponse;
+}
+
+
+describe("DeviceList management:", function() {
+    if (!global.Olm) {
+        console.warn('not running deviceList tests: Olm not present');
+        return;
+    }
+
+    let sessionStoreBackend;
+    let aliceTestClient;
+
+    async function createTestClient() {
+        const testClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs", sessionStoreBackend,
+        );
+        await testClient.client.initCrypto();
+        return testClient;
+    }
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        // we create our own sessionStoreBackend so that we can use it for
+        // another TestClient.
+        sessionStoreBackend = new testUtils.MockStorageApi();
+
+        aliceTestClient = await createTestClient();
+    });
+
+    afterEach(function() {
+        aliceTestClient.stop();
+    });
+
+    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
+        return aliceTestClient.start().then(function() {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+            return aliceTestClient.flushSync();
+        }).then(function() {
+            console.log("Forcing alice to download our device keys");
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {},
+                },
+            });
+
+            return Promise.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log("Telling alice to send a megolm message");
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/',
+            ).respond(200, {
+                    event_id: '$event_id',
+            });
+
+            return Promise.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
+            ]);
+        });
+    });
+
+
+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.flushSync();
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return Promise.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1),
+                   ),
+                   aliceTestClient.client._crypto._deviceList.saveIfDirty(),
+               ]);
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   expect(data.syncToken).toEqual(1);
+               });
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               // flush both syncs
+               return aliceTestClient.flushSync().then(() => {
+                   return aliceTestClient.flushSync();
+               });
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   if (bobStat != 1 && bobStat != 2) {
+                       throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                       bobStat);
+                   }
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
+                       );
+                   }
+               });
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   expect(bobStat).toEqual(3);
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
+                       );
+                   }
+               });
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   const chrisStat = data.trackingStatus['@bob:xyz'];
+
+                   expect(bobStat).toEqual(3);
+                   expect(chrisStat).toEqual(3);
+                   expect(data.syncToken).toEqual(3);
+               });
+           });
+       }).timeout(3000);
+
+    // https://github.com/vector-im/riot-web/issues/4983
+    describe("Alice should know she has stale device lists", () => {
+        beforeEach(async function() {
+            await aliceTestClient.start();
+
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, getSyncResponse(['@bob:xyz']));
+            await aliceTestClient.flushSync();
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                200, {
+                    device_keys: {
+                        '@bob:xyz': {},
+                    },
+                },
+            );
+            await aliceTestClient.httpBackend.flush('/keys/query', 1);
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toBeGreaterThan(
+                    0, "Alice should be tracking bob's device list",
+                );
+            });
+        });
+
+        it("when Bob leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        join: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Alice leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        leave: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Bob leaves whilst Alice is offline", async function() {
+            aliceTestClient.stop();
+
+            const anotherTestClient = await createTestClient();
+
+            try {
+                await anotherTestClient.start();
+                anotherTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, getSyncResponse([]));
+                await anotherTestClient.flushSync();
+                await anotherTestClient.client._crypto._deviceList.saveIfDirty();
+
+                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                    const bobStat = data.trackingStatus['@bob:xyz'];
+
+                    expect(bobStat).toEqual(
+                        0, "Alice should have marked bob's device list as untracked",
+                    );
+                });
+            } finally {
+                anotherTestClient.stop();
+            }
+        });
+    });
+});

spec/integ/matrix-client-crypto.spec.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -133,7 +134,7 @@ function expectAliClaimKeys() {
         // it can take a while to process the key query, so give it some extra
         // time, and make sure the claim actually happens rather than ploughing on
         // confusingly.
-        return aliTestClient.httpBackend.flush("/keys/claim", 1).then((r) => {
+        return aliTestClient.httpBackend.flush("/keys/claim", 1, 500).then((r) => {
             expect(r).toEqual(1, "Ali did not claim Bob's keys");
         });
     });
@@ -154,11 +155,15 @@ function aliDownloadsKeys() {
 
     // check that the localStorage is updated as we expect (not sure this is
     // an integration test, but meh)
-    return Promise.all([p1, p2]).then(function() {
-        const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
-        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
-        expect(devices[bobDeviceId].verified).
-            toBe(0); // DeviceVerification.UNVERIFIED
+    return Promise.all([p1, p2]).then(() => {
+        return aliTestClient.client._crypto._deviceList.saveIfDirty();
+    }).then(() => {
+        aliTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            const devices = data.devices[bobUserId];
+            expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
+            expect(devices[bobDeviceId].verified).
+                toBe(0); // DeviceVerification.UNVERIFIED
+        });
     });
 }
 
@@ -314,31 +319,37 @@ function recvMessage(httpBackend, client, sender, message) {
         },
     };
     httpBackend.when("GET", "/sync").respond(200, syncData);
-    const deferred = Promise.defer();
-    const onEvent = function(event) {
-        console.log(client.credentials.userId + " received event",
-                    event);
 
-        // ignore the m.room.member events
-        if (event.getType() == "m.room.member") {
-            return;
-        }
+    const eventPromise = new Promise((resolve, reject) => {
+        const onEvent = function(event) {
+            // ignore the m.room.member events
+            if (event.getType() == "m.room.member") {
+                return;
+            }
+            console.log(client.credentials.userId + " received event",
+                        event);
 
+            client.removeListener("event", onEvent);
+            resolve(event);
+        };
+        client.on("event", onEvent);
+    });
+
+    httpBackend.flush();
+
+    return eventPromise.then((event) => {
+        expect(event.isEncrypted()).toBeTruthy();
+
+        // it may still be being decrypted
+        return testUtils.awaitDecryption(event);
+    }).then((event) => {
         expect(event.getType()).toEqual("m.room.message");
         expect(event.getContent()).toEqual({
             msgtype: "m.text",
             body: "Hello, World",
         });
         expect(event.isEncrypted()).toBeTruthy();
-
-        client.removeListener("event", onEvent);
-        deferred.resolve();
-    };
-
-    client.on("event", onEvent);
-
-    httpBackend.flush();
-    return deferred.promise;
+    });
 }
 
 
@@ -583,27 +594,25 @@ describe("MatrixClient crypto", function() {
                 };
                 bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);
 
-                const deferred = Promise.defer();
-                const onEvent = function(event) {
-                    console.log(bobUserId + " received event",
-                                event);
-
-                    // ignore the m.room.member events
-                    if (event.getType() == "m.room.member") {
-                        return;
-                    }
-
-                    expect(event.isEncrypted()).toBeTruthy();
-
-                    expect(event.getType()).toEqual("m.room.message");
-                    expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
-                    deferred.resolve();
-                };
-
-                bobTestClient.client.once("event", onEvent);
+                const eventPromise = new Promise((resolve, reject) => {
+                    const onEvent = function(event) {
+                        console.log(bobUserId + " received event",
+                                    event);
+                        resolve(event);
+                    };
+                    bobTestClient.client.once("event", onEvent);
+                });
 
                 bobTestClient.httpBackend.flush();
-                return deferred.promise;
+                return eventPromise;
+            }).then((event) => {
+                expect(event.isEncrypted()).toBeTruthy();
+
+                // it may still be being decrypted
+                return testUtils.awaitDecryption(event);
+            }).then((event) => {
+                expect(event.getType()).toEqual("m.room.message");
+                expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
             });
     });
 

spec/integ/matrix-client-syncing.spec.js

@@ -437,6 +437,34 @@ describe("MatrixClient syncing", function() {
             });
         });
 
+        // XXX: This test asserts that the js-sdk obeys the spec and treats state
+        // events that arrive in the incremental sync as if they preceeded the
+        // timeline events, however this breaks peeking, so it's disabled
+        // (see sync.js)
+        xit("should correctly interpret state in incremental sync.", function() {
+            httpBackend.when("GET", "/sync").respond(200, syncData);
+            httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+            client.startClient();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
+                const stateAtStart = room.getLiveTimeline().getState(
+                    EventTimeline.BACKWARDS,
+                );
+                const startRoomNameEvent = stateAtStart.getStateEvents('m.room.name', '');
+                expect(startRoomNameEvent.getContent().name).toEqual('Old room name');
+
+                const stateAtEnd = room.getLiveTimeline().getState(
+                    EventTimeline.FORWARDS,
+                );
+                const endRoomNameEvent = stateAtEnd.getStateEvents('m.room.name', '');
+                expect(endRoomNameEvent.getContent().name).toEqual('A new room name');
+            });
+        });
+
         xit("should update power levels for users in a room", function() {
 
         });

spec/integ/megolm-integ.spec.js

@@ -119,6 +119,7 @@ function encryptMegolmEvent(opts) {
     }
 
     return {
+        event_id: 'test_megolm_event',
         content: {
             algorithm: "m.megolm.v1.aes-sha2",
             ciphertext: opts.groupSession.encrypt(JSON.stringify(plaintext)),
@@ -342,6 +343,9 @@ describe("megolm", function() {
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.isEncrypted()).toBe(true);
+            return testUtils.awaitDecryption(event);
+        }).then((event) => {
             expect(event.getContent().body).toEqual('42');
         });
     });
@@ -405,6 +409,18 @@ describe("megolm", function() {
         }).then(function() {
             const room = aliceTestClient.client.getRoom(ROOM_ID);
             const event = room.getLiveTimeline().getEvents()[0];
+
+            if (event.getContent().msgtype != 'm.bad.encrypted') {
+                return event;
+            }
+
+            return new Promise((resolve, reject) => {
+                event.once('Event.decrypted', (ev) => {
+                    console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    resolve(ev);
+                });
+            });
+        }).then((event) => {
             expect(event.getContent().body).toEqual('42');
         });
     });
@@ -564,46 +580,6 @@ describe("megolm", function() {
         });
     });
 
-    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-
-            return aliceTestClient.flushSync();
-        }).then(function() {
-            console.log("Forcing alice to download our device keys");
-
-            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
-                device_keys: {
-                    '@bob:xyz': {},
-                },
-            });
-
-            return Promise.all([
-                aliceTestClient.client.downloadKeys(['@bob:xyz']),
-                aliceTestClient.httpBackend.flush('/keys/query', 1),
-            ]);
-        }).then(function() {
-            console.log("Telling alice to send a megolm message");
-
-            aliceTestClient.httpBackend.when(
-                'PUT', '/send/',
-            ).respond(200, {
-                    event_id: '$event_id',
-            });
-
-            return Promise.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-
-                // the crypto stuff can take a while, so give the requests a whole second.
-                aliceTestClient.httpBackend.flushAllExpected({
-                    timeout: 1000,
-                }),
-            ]);
-        });
-    });
-
-
     it("We shouldn't attempt to send to blocked devices", function() {
         return aliceTestClient.start().then(() => {
             // establish an olm session with alice
@@ -901,128 +877,6 @@ describe("megolm", function() {
     });
 
 
-    it("We should not get confused by out-of-order device query responses",
-       () => {
-           // https://github.com/vector-im/riot-web/issues/3126
-           return aliceTestClient.start().then(() => {
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(
-                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
-               return aliceTestClient.flushSync();
-           }).then(() => {
-               // to make sure the initial device queries are flushed out, we
-               // attempt to send a message.
-
-               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
-                   200, {
-                       device_keys: {
-                           '@bob:xyz': {},
-                           '@chris:abc': {},
-                       },
-                   },
-               );
-
-               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
-                   200, {event_id: '$event1'});
-
-               return Promise.all([
-                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
-                       () => aliceTestClient.httpBackend.flush('/send/', 1),
-                   ),
-               ]);
-           }).then(() => {
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
-
-               // invalidate bob's and chris's device lists in separate syncs
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '2',
-                   device_lists: {
-                       changed: ['@bob:xyz'],
-                   },
-               });
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '3',
-                   device_lists: {
-                       changed: ['@chris:abc'],
-                   },
-               });
-               // flush both syncs
-               return aliceTestClient.flushSync().then(() => {
-                   return aliceTestClient.flushSync();
-               });
-           }).then(() => {
-               // check that we don't yet have a request for chris's devices.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@chris:abc': {},
-                   },
-                   token: '3',
-               }).respond(200, {
-                   device_keys: {'@chris:abc': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(0);
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               if (bobStat != 1 && bobStat != 2) {
-                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   chrisStat);
-               }
-
-               // now add an expectation for a query for bob's devices, and let
-               // it complete.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@bob:xyz': {},
-                   },
-                   token: '2',
-               }).respond(200, {
-                   device_keys: {'@bob:xyz': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               expect(bobStat).toEqual(3);
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               // now let the query for chris's devices complete.
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@chris:abc']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-
-               expect(bobStat).toEqual(3);
-               expect(chrisStat).toEqual(3);
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
-           });
-       });
-
     it("Alice exports megolm keys and imports them to a new device", function() {
         let messageEncrypted;
 

spec/test-utils.js

@@ -219,3 +219,25 @@ module.exports.MockStorageApi.prototype = {
         delete this.data[k];
     },
 };
+
+
+/**
+ * If an event is being decrypted, wait for it to finish being decrypted.
+ *
+ * @param {MatrixEvent} event
+ * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
+ */
+module.exports.awaitDecryption = function(event) {
+    if (!event.isBeingDecrypted()) {
+        return Promise.resolve(event);
+    }
+
+    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+
+    return new Promise((resolve, reject) => {
+        event.once('Event.decrypted', (ev) => {
+            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            resolve(ev);
+        });
+    });
+};

spec/unit/crypto/DeviceList.spec.js

@@ -1,6 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import DeviceList from '../../../lib/crypto/DeviceList';
 import MockStorageApi from '../../MockStorageApi';
 import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
 import testUtils from '../../test-utils';
 import utils from '../../../lib/utils';
 
@@ -40,6 +58,7 @@ const signedDeviceList = {
 describe('DeviceList', function() {
     let downloadSpy;
     let sessionStore;
+    let cryptoStore;
 
     beforeEach(function() {
         testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
@@ -47,6 +66,7 @@ describe('DeviceList', function() {
         downloadSpy = expect.createSpy();
         const mockStorage = new MockStorageApi();
         sessionStore = new WebStorageSessionStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore();
     });
 
     function createTestDeviceList() {
@@ -56,7 +76,7 @@ describe('DeviceList', function() {
         const mockOlm = {
             verifySignature: function(key, message, signature) {},
         };
-        return new DeviceList(baseApis, sessionStore, mockOlm);
+        return new DeviceList(baseApis, cryptoStore, sessionStore, mockOlm);
     }
 
     it("should successfully download and store device keys", function() {
@@ -72,7 +92,7 @@ describe('DeviceList', function() {
         queryDefer1.resolve(utils.deepCopy(signedDeviceList));
 
         return prom1.then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
         });
     });
@@ -97,14 +117,15 @@ describe('DeviceList', function() {
         dl.invalidateUserDeviceList('@test1:sw1v.org');
         dl.refreshOutdatedDeviceLists();
 
-        // the first request completes
-        queryDefer1.resolve({
-            device_keys: {
-                '@test1:sw1v.org': {},
-            },
-        });
-
-        return prom1.then(() => {
+        dl.saveIfDirty().then(() => {
+            // the first request completes
+            queryDefer1.resolve({
+                device_keys: {
+                    '@test1:sw1v.org': {},
+                },
+            });
+            return prom1;
+        }).then(() => {
             // uh-oh; user restarts before second request completes. The new instance
             // should know we never got a complete device list.
             console.log("Creating new devicelist to simulate app reload");
@@ -121,7 +142,7 @@ describe('DeviceList', function() {
             // allow promise chain to complete
             return prom3;
         }).then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
         });
     });

spec/unit/crypto/algorithms/megolm.spec.js

@@ -10,6 +10,7 @@ import Promise from 'bluebird';
 import sdk from '../../../..';
 import algorithms from '../../../../lib/crypto/algorithms';
 import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
 import MockStorageApi from '../../../MockStorageApi';
 import testUtils from '../../../test-utils';
 
@@ -45,8 +46,9 @@ describe("MegolmDecryption", function() {
 
         const mockStorage = new MockStorageApi();
         const sessionStore = new WebStorageSessionStore(mockStorage);
+        const cryptoStore = new MemoryCryptoStore(mockStorage);
 
-        const olmDevice = new OlmDevice(sessionStore);
+        const olmDevice = new OlmDevice(sessionStore, cryptoStore);
 
         megolmDecryption = new MegolmDecryption({
             userId: '@user:id',
@@ -71,9 +73,13 @@ describe("MegolmDecryption", function() {
             groupSession = new global.Olm.OutboundGroupSession();
             groupSession.create();
 
-            const event = new MatrixEvent({});
-            event.setClearData(
-                {
+            // construct a fake decrypted key event via the use of a mocked
+            // 'crypto' implementation.
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+            });
+            const decryptedData = {
+                clearEvent: {
                     type: 'm.room_key',
                     content: {
                         algorithm: 'm.megolm.v1.aes-sha2',
@@ -82,11 +88,19 @@ describe("MegolmDecryption", function() {
                         session_key: groupSession.session_key(),
                     },
                 },
-                "SENDER_CURVE25519",
-                "SENDER_ED25519",
-            );
+                senderCurve25519Key: "SENDER_CURVE25519",
+                claimedEd25519Key: "SENDER_ED25519",
+            };
 
-            megolmDecryption.onRoomKeyEvent(event);
+            const mockCrypto = {
+                decryptEvent: function() {
+                    return Promise.resolve(decryptedData);
+                },
+            };
+
+            return event.attemptDecryption(mockCrypto).then(() => {
+                megolmDecryption.onRoomKeyEvent(event);
+            });
         });
 
         it('can decrypt an event', function() {
@@ -104,8 +118,8 @@ describe("MegolmDecryption", function() {
                 },
             });
 
-            return megolmDecryption.decryptEvent(event).then(() => {
-                expect(event.getContent()).toEqual('testytest');
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
             });
         });
 
@@ -129,12 +143,16 @@ describe("MegolmDecryption", function() {
                 const deviceInfo = {};
                 mockCrypto.getStoredDevice.andReturn(deviceInfo);
 
-                const awaitEnsureSessions = new Promise((res, rej) => {
-                    mockOlmLib.ensureOlmSessionsForDevices.andCall(() => {
+                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
+                    Promise.resolve({'@alice:foo': {'alidevice': {
+                        sessionId: 'alisession',
+                    }}}),
+                );
+
+                const awaitEncryptForDevice = new Promise((res, rej) => {
+                    mockOlmLib.encryptMessageForDevice.andCall(() => {
                         res();
-                        return Promise.resolve({'@alice:foo': {'alidevice': {
-                            sessionId: 'alisession',
-                        }}});
+                        return Promise.resolve();
                     });
                 });
 
@@ -144,7 +162,7 @@ describe("MegolmDecryption", function() {
                 megolmDecryption.shareKeysWithDevice(keyRequest);
 
                 // it's asynchronous, so we have to wait a bit
-                return awaitEnsureSessions;
+                return awaitEncryptForDevice;
             }).then(() => {
                 // check that it called encryptMessageForDevice with
                 // appropriate args.
@@ -165,5 +183,88 @@ describe("MegolmDecryption", function() {
                 expect(payload.content.session_key).toExist();
             });
         });
+
+        it("can detect replay attacks", function() {
+            // trying to decrypt two different messages (marked by different
+            // event IDs or timestamps) using the same (sender key, session id,
+            // message index) triple should result in an exception being thrown
+            // as it should be detected as a replay attack.
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event1 = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            const successHandler = expect.createSpy();
+            const failureHandler = expect.createSpy()
+                .andCall((err) => {
+                    expect(err.toString()).toMatch(
+                        /Duplicate message index, possible replay attack/,
+                    );
+                });
+
+            return megolmDecryption.decryptEvent(event1).then((res) => {
+                const event2 = new MatrixEvent({
+                    type: 'm.room.encrypted',
+                    room_id: ROOM_ID,
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        sender_key: "SENDER_CURVE25519",
+                        session_id: sessionId,
+                        ciphertext: cipherText,
+                    },
+                    event_id: "$event2",
+                    origin_server_ts: 1507754149000,
+                });
+
+                return megolmDecryption.decryptEvent(event2);
+            }).then(
+                successHandler,
+                failureHandler,
+            ).then(() => {
+                expect(successHandler).toNotHaveBeenCalled();
+                expect(failureHandler).toHaveBeenCalled();
+            });
+        });
+
+        it("allows re-decryption of the same event", function() {
+            // in contrast with the previous test, if the event ID and
+            // timestamp are the same, then it should not be considered a
+            // replay attack
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                return megolmDecryption.decryptEvent(event);
+                // test is successful if no exception is thrown
+            });
+        });
     });
 });

spec/unit/event.spec.js

@@ -0,0 +1,82 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import sdk from '../..';
+const MatrixEvent = sdk.MatrixEvent;
+
+import testUtils from '../test-utils';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+describe("MatrixEvent", () => {
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+    });
+
+    describe(".attemptDecryption", () => {
+        let encryptedEvent;
+
+        beforeEach(() => {
+            encryptedEvent = new MatrixEvent({
+                id: 'test_encrypted_event',
+                type: 'm.room.encrypted',
+                content: {
+                    ciphertext: 'secrets',
+                },
+            });
+        });
+
+        it('should retry decryption if a retry is queued', () => {
+            let callCount = 0;
+
+            let prom2;
+
+            const crypto = {
+                decryptEvent: function() {
+                    ++callCount;
+                    console.log(`decrypt: ${callCount}`);
+                    if (callCount == 1) {
+                        // schedule a second decryption attempt while
+                        // the first one is still running.
+                        prom2 = encryptedEvent.attemptDecryption(crypto);
+
+                        const error = new Error("nope");
+                        error.name = 'DecryptionError';
+                        return Promise.reject(error);
+                    } else {
+                        expect(prom2.isFulfilled()).toBe(
+                            false, 'second attemptDecryption resolved too soon');
+
+                        return Promise.resolve({
+                            clearEvent: {
+                                type: 'm.room.message',
+                            },
+                        });
+                    }
+                },
+            };
+
+            return encryptedEvent.attemptDecryption(crypto).then(() => {
+                expect(callCount).toEqual(2);
+                expect(encryptedEvent.getType()).toEqual('m.room.message');
+
+                // make sure the second attemptDecryption resolves
+                return prom2;
+            });
+        });
+    });
+});

spec/unit/matrix-client.spec.js

@@ -132,11 +132,12 @@ describe("MatrixClient", function() {
         ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
         store = [
             "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "save", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
             "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
             "getSyncAccumulator", "startup", "deleteAllData",
         ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
         store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
         store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",

spec/unit/pushprocessor.spec.js

@@ -24,6 +24,9 @@ describe('NotificationService', function() {
                             name: testDisplayName,
                         };
                     },
+                    getJoinedMemberCount: function() {
+                        return 0;
+                    },
                     members: {},
                 },
             };

spec/unit/room.spec.js

@@ -383,22 +383,25 @@ describe("Room", function() {
     });
 
     const resetTimelineTests = function(timelineSupport) {
-        const events = [
-            utils.mkMessage({
-                room: roomId, user: userA, msg: "A message", event: true,
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "New Room Name" },
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "Another New Name" },
-            }),
-        ];
+        let events = null;
 
         beforeEach(function() {
             room = new Room(roomId, {timelineSupport: timelineSupport});
+            // set events each time to avoid resusing Event objects (which
+            // doesn't work because they get frozen)
+            events = [
+                utils.mkMessage({
+                    room: roomId, user: userA, msg: "A message", event: true,
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "New Room Name" },
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "Another New Name" },
+                }),
+            ];
         });
 
         it("should copy state from previous timeline", function() {
@@ -886,7 +889,7 @@ describe("Room", function() {
                 expect(name).toEqual("Empty room");
             });
 
-            it("should return 'Invite from [inviter display name] if state event " +
+            it("should return '[inviter display name] if state event " +
                "available",
             function() {
                 setJoinRule("invite");
@@ -894,7 +897,7 @@ describe("Room", function() {
                 addMember(userB, "invite", {user: userA});
                 room.recalculate(userB);
                 const name = room.name;
-                expect(name).toEqual("Invite from Alice");
+                expect(name).toEqual("Alice");
             });
 
             it("should return inviter mxid if display name not available",
@@ -904,7 +907,7 @@ describe("Room", function() {
                 addMember(userB, "invite", {user: userA});
                 room.recalculate(userB);
                 const name = room.name;
-                expect(name).toEqual("Invite from " + userA);
+                expect(name).toEqual(userA);
             });
         });
     });

src/ReEmitter.js

@@ -0,0 +1,46 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module
+ */
+
+export default class Reemitter {
+    constructor(target) {
+        this.target = target;
+
+        // We keep one bound event handler for each event name so we know
+        // what event is arriving
+        this.boundHandlers = {};
+    }
+
+    _handleEvent(eventName, ...args) {
+        this.target.emit(eventName, ...args);
+    }
+
+    reEmit(source, eventNames) {
+        for (const eventName of eventNames) {
+            if (this.boundHandlers[eventName] === undefined) {
+                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
+            }
+            const boundHandler = this.boundHandlers[eventName];
+
+            source.on(eventName, boundHandler);
+        }
+    }
+}

src/base-apis.js

@@ -89,9 +89,14 @@ MatrixBaseApis.prototype.getHomeserverUrl = function() {
 
 /**
  * Get the Identity Server URL of this client
+ * @param {boolean} stripProto whether or not to strip the protocol from the URL
  * @return {string} Identity Server URL of this client
  */
-MatrixBaseApis.prototype.getIdentityServerUrl = function() {
+MatrixBaseApis.prototype.getIdentityServerUrl = function(stripProto=false) {
+    if (stripProto && (this.idBaseUrl.startsWith("http://") ||
+            this.idBaseUrl.startsWith("https://"))) {
+        return this.idBaseUrl.split("://")[1];
+    }
     return this.idBaseUrl;
 };
 
@@ -411,6 +416,16 @@ MatrixBaseApis.prototype.getGroupSummary = function(groupId) {
     return this._http.authedRequest(undefined, "GET", path);
 };
 
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group profile object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupProfile = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/profile", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
 /**
  * @param {string} groupId
  * @param {Object} profile The group profile object
@@ -428,6 +443,27 @@ MatrixBaseApis.prototype.setGroupProfile = function(groupId, profile) {
     );
 };
 
+/**
+ * @param {string} groupId
+ * @param {object} policy The join policy for the group. Must include at
+ *     least a 'type' field which is 'open' if anyone can join the group
+ *     the group without prior approval, or 'invite' if an invite is
+ *     required to join.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupJoinPolicy = function(groupId, policy) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/settings/m.join_policy",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(
+        undefined, "PUT", path, undefined, {
+            'm.join_policy': policy,
+        },
+    );
+};
+
 /**
  * @param {string} groupId
  * @return {module:client.Promise} Resolves: Group users list object
@@ -438,6 +474,212 @@ MatrixBaseApis.prototype.getGroupUsers = function(groupId) {
     return this._http.authedRequest(undefined, "GET", path);
 };
 
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupInvitedUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/invited_users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group rooms list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupRooms = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/rooms", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.inviteUserToGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/invite/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/remove/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @param {string} roleId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addUserToGroupSummary = function(groupId, userId, roleId) {
+    const path = utils.encodeUri(
+        roleId ?
+            "/groups/$groupId/summary/$roleId/users/$userId" :
+            "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $roleId: roleId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroupSummary = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {string} categoryId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroupSummary = function(groupId, roomId, categoryId) {
+    const path = utils.encodeUri(
+        categoryId ?
+            "/groups/$groupId/summary/$categoryId/rooms/$roomId" :
+            "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $categoryId: categoryId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroupSummary = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroup = function(groupId, roomId, isPublic) {
+    if (isPublic === undefined) {
+        isPublic = true;
+    }
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { "m.visibility": { type: isPublic ? "public" : "private" } },
+    );
+};
+
+/**
+ * Configure the visibility of a room-group association.
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.updateGroupRoomVisibility = function(groupId, roomId, isPublic) {
+    // NB: The /config API is generic but there's not much point in exposing this yet as synapse
+    //     is the only server to implement this. In future we should consider an API that allows
+    //     arbitrary configuration, i.e. "config/$configKey".
+
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId/config/m.visibility",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { type: isPublic ? "public" : "private" },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroup = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {Object} opts Additional options to send alongside the acceptance.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.acceptGroupInvite = function(groupId, opts = null) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/accept_invite",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, opts || {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.joinGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/join",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.leaveGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/leave",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
 /**
  * @return {module:client.Promise} Resolves: The groups to which the user is joined
  * @return {module:http-api.MatrixError} Rejects: with an error response.
@@ -461,6 +703,42 @@ MatrixBaseApis.prototype.createGroup = function(content) {
     );
 };
 
+/**
+ * @param {string[]} userIds List of user IDs
+ * @return {module:client.Promise} Resolves: Object as exmaple below
+ *
+ *     {
+ *         "users": {
+ *             "@bob:example.com": {
+ *                 "+example:example.com"
+ *             }
+ *         }
+ *     }
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getPublicisedGroups = function(userIds) {
+    const path = utils.encodeUri("/publicised_groups");
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, { user_ids: userIds },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {bool} isPublic Whether the user's membership of this group is made public
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupPublicity = function(groupId, isPublic) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/update_publicity",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {
+        publicise: isPublic,
+    });
+};
+
 /**
  * Retrieve a state event.
  * @param {string} roomId
@@ -991,6 +1269,27 @@ MatrixBaseApis.prototype.deleteDevice = function(device_id, auth) {
     );
 };
 
+/**
+ * Delete multiple device
+ *
+ * @param {string[]} devices IDs of the devices to delete
+ * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
+ * @return {module:client.Promise} Resolves: result object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.deleteMultipleDevices = function(devices, auth) {
+    const body = {devices};
+
+    if (auth) {
+        body.auth = auth;
+    }
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "POST", "/delete_devices", undefined, body,
+        httpApi.PREFIX_UNSTABLE,
+    );
+};
+
 
 // Push operations
 // ===============

src/client.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -40,7 +41,8 @@ const SyncApi = require("./sync");
 const MatrixBaseApis = require("./base-apis");
 const MatrixError = httpApi.MatrixError;
 
-import reEmit from './reemit';
+import ReEmitter from './ReEmitter';
+import RoomList from './crypto/RoomList';
 
 const SCROLLBACK_DELAY_MS = 3000;
 let CRYPTO_ENABLED = false;
@@ -113,8 +115,20 @@ try {
  *    crypto store implementation.
  */
 function MatrixClient(opts) {
+    // Allow trailing slash in HS url
+    if (opts.baseUrl && opts.baseUrl.endsWith("/")) {
+        opts.baseUrl = opts.baseUrl.substr(0, opts.baseUrl.length - 1);
+    }
+
+    // Allow trailing slash in IS url
+    if (opts.idBaseUrl && opts.idBaseUrl.endsWith("/")) {
+        opts.idBaseUrl = opts.idBaseUrl.substr(0, opts.idBaseUrl.length - 1);
+    }
+
     MatrixBaseApis.call(this, opts);
 
+    this.reEmitter = new ReEmitter(this);
+
     this.store = opts.store || new StubStore();
 
     this.deviceId = opts.deviceId || null;
@@ -163,9 +177,19 @@ function MatrixClient(opts) {
     this._cryptoStore = opts.cryptoStore;
     this._sessionStore = opts.sessionStore;
 
+    this._forceTURN = opts.forceTURN || false;
+
     if (CRYPTO_ENABLED) {
         this.olmVersion = Crypto.getOlmVersion();
     }
+
+    // List of which rooms have encryption enabled: separate from crypto because
+    // we still want to know which rooms are encrypted even if crypto is disabled:
+    // we don't want to start sending unencrypted events to them.
+    this._roomList = new RoomList(this._cryptoStore, this._sessionStore);
+
+    // The pushprocessor caches useful things, so keep one and re-use it
+    this._pushProcessor = new PushProcessor(this);
 }
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
@@ -240,6 +264,16 @@ MatrixClient.prototype.supportsVoip = function() {
     return this._supportsVoip;
 };
 
+/**
+ * Set whether VoIP calls are forced to use only TURN
+ * candidates. This is the same as the forceTURN option
+ * when creating the client.
+ * @param {bool} forceTURN True to force use of TURN servers
+ */
+MatrixClient.prototype.setForceTURN = function(forceTURN) {
+    this._forceTURN = forceTURN;
+};
+
 /**
  * Get the current sync state.
  * @return {?string} the sync state, which may be null.
@@ -326,13 +360,6 @@ MatrixClient.prototype.initCrypto = async function() {
         return;
     }
 
-    if (!CRYPTO_ENABLED) {
-        throw new Error(
-            `End-to-end encryption not supported in this js-sdk build: did ` +
-                `you remember to load the olm library?`,
-        );
-    }
-
     if (!this._sessionStore) {
         // this is temporary, the sessionstore is supposed to be going away
         throw new Error(`Cannot enable encryption: no sessionStore provided`);
@@ -342,6 +369,16 @@ MatrixClient.prototype.initCrypto = async function() {
         throw new Error(`Cannot enable encryption: no cryptoStore provided`);
     }
 
+    // initialise the list of encrypted rooms (whether or not crypto is enabled)
+    await this._roomList.init();
+
+    if (!CRYPTO_ENABLED) {
+        throw new Error(
+            `End-to-end encryption not supported in this js-sdk build: did ` +
+                `you remember to load the olm library?`,
+        );
+    }
+
     const userId = this.getUserId();
     if (userId === null) {
         throw new Error(
@@ -362,11 +399,13 @@ MatrixClient.prototype.initCrypto = async function() {
         userId, this.deviceId,
         this.store,
         this._cryptoStore,
+        this._roomList,
     );
 
-    reEmit(this, crypto, [
+    this.reEmitter.reEmit(crypto, [
         "crypto.roomKeyRequest",
         "crypto.roomKeyRequestCancellation",
+        "crypto.warning",
     ]);
 
     await crypto.init();
@@ -531,11 +570,10 @@ async function _setDeviceVerification(
 
 /**
  * Set the global override for whether the client should ever send encrypted
- * messages to unverified devices.  If false, it can still be overridden
- * per-room.  If true, it overrides the per-room settings.
+ * messages to unverified devices.  This provides the default for rooms which
+ * do not specify a value.
  *
- * @param {boolean} value whether to unilaterally blacklist all
- * unverified devices
+ * @param {boolean} value whether to blacklist all unverified devices by default
  */
 MatrixClient.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
     if (this._crypto === null) {
@@ -545,8 +583,7 @@ MatrixClient.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
 };
 
 /**
- * @return {boolean} whether to unilaterally blacklist all
- * unverified devices
+ * @return {boolean} whether to blacklist all unverified devices by default
  */
 MatrixClient.prototype.getGlobalBlacklistUnverifiedDevices = function() {
     if (this._crypto === null) {
@@ -586,6 +623,16 @@ MatrixClient.prototype.isEventSenderVerified = async function(event) {
     return device.isVerified();
 };
 
+/**
+ * Cancel a room key request for this event if one is ongoing and resend the
+ * request.
+ * @param  {MatrxEvent} event event of which to cancel and resend the room
+ *                            key request.
+ */
+MatrixClient.prototype.cancelAndResendEventRoomKeyRequest = function(event) {
+    event.cancelAndResendKeyRequest(this._crypto);
+};
+
 /**
  * Enable end-to-end encryption for a room.
  * @param {string} roomId The room ID to enable encryption in.
@@ -622,11 +669,7 @@ MatrixClient.prototype.isRoomEncrypted = function(roomId) {
     // we don't have an m.room.encrypted event, but that might be because
     // the server is hiding it from us. Check the store to see if it was
     // previously encrypted.
-    if (!this._sessionStore) {
-        return false;
-    }
-
-    return Boolean(this._sessionStore.getEndToEndRoom(roomId));
+    return this._roomList.isRoomEncrypted(roomId);
 };
 
 /**
@@ -659,6 +702,30 @@ MatrixClient.prototype.importRoomKeys = function(keys) {
     return this._crypto.importRoomKeys(keys);
 };
 
+// Group ops
+// =========
+// Operations on groups that come down the sync stream (ie. ones the
+// user is a member of or invited to)
+
+/**
+ * Get the group for the given group ID.
+ * This function will return a valid group for any group for which a Group event
+ * has been emitted.
+ * @param {string} groupId The group ID
+ * @return {Group} The Group or null if the group is not known or there is no data store.
+ */
+MatrixClient.prototype.getGroup = function(groupId) {
+    return this.store.getGroup(groupId);
+};
+
+/**
+ * Retrieve all known groups.
+ * @return {Groups[]} A list of groups, or an empty list if there is no data store.
+ */
+MatrixClient.prototype.getGroups = function() {
+    return this.store.getGroups();
+};
+
 // Room ops
 // ========
 
@@ -724,13 +791,44 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
 /**
  * Get account data event of given type for the current user.
  * @param {string} eventType The event type
- * @param {module:client.callback} callback Optional.
  * @return {?object} The contents of the given account data event
  */
 MatrixClient.prototype.getAccountData = function(eventType) {
     return this.store.getAccountData(eventType);
 };
 
+/**
+ * Gets the users that are ignored by this client
+ * @returns {string[]} The array of users that are ignored (empty if none)
+ */
+MatrixClient.prototype.getIgnoredUsers = function() {
+    const event = this.getAccountData("m.ignored_user_list");
+    if (!event || !event.getContent() || !event.getContent()["ignored_users"]) return [];
+    return Object.keys(event.getContent()["ignored_users"]);
+};
+
+/**
+ * Sets the users that the current user should ignore.
+ * @param {string[]} userIds the user IDs to ignore
+ * @param {module:client.callback} [callback] Optional.
+ * @return {module:client.Promise} Resolves: Account data event
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.setIgnoredUsers = function(userIds, callback) {
+    const content = {ignored_users: {}};
+    userIds.map((u) => content.ignored_users[u] = {});
+    return this.setAccountData("m.ignored_user_list", content, callback);
+};
+
+/**
+ * Gets whether or not a specific user is being ignored by this client.
+ * @param {string} userId the user ID to check
+ * @returns {boolean} true if the user is ignored, false otherwise
+ */
+MatrixClient.prototype.isUserIgnored = function(userId) {
+    return this.getIgnoredUsers().indexOf(userId) !== -1;
+};
+
 // Room operations
 // ===============
 
@@ -1057,8 +1155,14 @@ function _sendEvent(client, room, event, callback) {
         console.error("Error sending event", err.stack || err);
 
         try {
-            _updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            // set the error on the event before we update the status:
+            // updating the status emits the event, so the state should be
+            // consistent at that point.
             event.error = err;
+            _updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            // also put the event object on the error: the caller will need this
+            // to resend or cancel the event
+            err.event = event;
 
             if (callback) {
                 callback(err);
@@ -1239,6 +1343,32 @@ MatrixClient.prototype.sendImageMessage = function(roomId, url, info, text, call
     return this.sendMessage(roomId, content, callback);
 };
 
+/**
+ * @param {string} roomId
+ * @param {string} url
+ * @param {Object} info
+ * @param {string} text
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: TODO
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.sendStickerMessage = function(roomId, url, info, text, callback) {
+    if (utils.isFunction(text)) {
+        callback = text; text = undefined;
+    }
+    if (!text) {
+        text = "Sticker";
+    }
+    const content = {
+         url: url,
+         info: info,
+         body: text,
+    };
+    return this.sendEvent(
+        roomId, "m.room.sticker", content, callback, undefined,
+    );
+};
+
 /**
  * @param {string} roomId
  * @param {string} body
@@ -1465,18 +1595,13 @@ MatrixClient.prototype.inviteByThreePid = function(roomId, medium, address, call
         { $roomId: roomId },
     );
 
-    let identityServerUrl = this.getIdentityServerUrl();
+    const identityServerUrl = this.getIdentityServerUrl(true);
     if (!identityServerUrl) {
         return Promise.reject(new MatrixError({
             error: "No supplied identity server URL",
             errcode: "ORG.MATRIX.JSSDK_MISSING_PARAM",
         }));
     }
-    if (identityServerUrl.indexOf("http://") === 0 ||
-            identityServerUrl.indexOf("https://") === 0) {
-        // this request must not have the protocol part because reasons
-        identityServerUrl = identityServerUrl.split("://")[1];
-    }
 
     return this._http.authedRequest(callback, "POST", path, undefined, {
         id_server: identityServerUrl,
@@ -1636,8 +1761,7 @@ function _membershipChange(client, roomId, userId, membership, reason, callback)
  */
 MatrixClient.prototype.getPushActionsForEvent = function(event) {
     if (!event.getPushActions()) {
-        const pushProcessor = new PushProcessor(this);
-        event.setPushActions(pushProcessor.actionsForEvent(event));
+        event.setPushActions(this._pushProcessor.actionsForEvent(event));
     }
     return event.getPushActions();
 };
@@ -2915,6 +3039,9 @@ MatrixClient.prototype.getTurnServers = function() {
  *
  * @param {Filter=} opts.filter The filter to apply to /sync calls. This will override
  * the opts.initialSyncLimit, which would normally result in a timeline limit filter.
+ *
+ * @param {Boolean=} opts.disablePresence True to perform syncing without automatically
+ * updating presence.
  */
 MatrixClient.prototype.startClient = function(opts) {
     if (this.clientRunning) {
@@ -3029,6 +3156,10 @@ function setupCallEventHandler(client) {
             // now loop through the buffer chronologically and inject them
             callEventBuffer.forEach(function(e) {
                 if (ignoreCallIds[e.getContent().call_id]) {
+                    console.log(
+                        'Ignoring previously answered/hungup call ' +
+                            e.getContent().call_id + ' (' + e.getId() + ')',
+                    );
                     return;
                 }
                 callEventHandler(e);
@@ -3037,20 +3168,25 @@ function setupCallEventHandler(client) {
         }
     });
 
-    client.on("event", function(event) {
-        if (!isClientPrepared) {
-            if (event.getType().indexOf("m.call.") === 0) {
-                callEventBuffer.push(event);
+    client.on("event", onEvent);
+
+    function onEvent(event) {
+        if (event.getType().indexOf("m.call.") !== 0) {
+            // not a call event
+            if (event.isBeingDecrypted() || event.isDecryptionFailure()) {
+                // not *yet* a call event, but might become one...
+                event.once("Event.decrypted", onEvent);
             }
             return;
         }
+        if (!isClientPrepared) {
+            callEventBuffer.push(event);
+            return;
+        }
         callEventHandler(event);
-    });
+    }
 
     function callEventHandler(event) {
-        if (event.getType().indexOf("m.call.") !== 0) {
-            return; // not a call event
-        }
         const content = event.getContent();
         let call = content.call_id ? client.callList[content.call_id] : undefined;
         let i;
@@ -3076,7 +3212,9 @@ function setupCallEventHandler(client) {
                 );
             }
 
-            call = webRtcCall.createNewMatrixCall(client, event.getRoomId());
+            call = webRtcCall.createNewMatrixCall(client, event.getRoomId(), {
+                forceTURN: client._forceTURN,
+            });
             if (!call) {
                 console.log(
                     "Incoming call ID " + content.call_id + " but this client " +
@@ -3242,6 +3380,9 @@ function _PojoToMatrixEventMapper(client) {
     function mapper(plainOldJsObject) {
         const event = new MatrixEvent(plainOldJsObject);
         if (event.isEncrypted()) {
+            client.reEmitter.reEmit(event, [
+                "Event.decrypted",
+            ]);
             event.attemptDecryption(client._crypto);
         }
         return event;
@@ -3423,6 +3564,17 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
  * });
  */
 
+ /**
+ * Fires whenever the sdk learns about a new group. <strong>This event
+ * is experimental and may change.</strong>
+ * @event module:client~MatrixClient#"Group"
+ * @param {Group} group The newly created, fully populated group.
+ * @example
+ * matrixClient.on("Group", function(group){
+ *   var groupId = group.groupId;
+ * });
+ */
+
  /**
  * Fires whenever a new Room is added. This will fire when you are invited to a
  * room, as well as when you join a room. <strong>This event is experimental and

src/crypto/DeviceList.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,9 +26,32 @@ import Promise from 'bluebird';
 
 import DeviceInfo from './deviceinfo';
 import olmlib from './olmlib';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+
+/* State transition diagram for DeviceList._deviceTrackingStatus
+ *
+ *                                |
+ *     stopTrackingDeviceList     V
+ *   +---------------------> NOT_TRACKED
+ *   |                            |
+ *   +<--------------------+      | startTrackingDeviceList
+ *   |                     |      V
+ *   |   +-------------> PENDING_DOWNLOAD <--------------------+-+
+ *   |   |                      ^ |                            | |
+ *   |   | restart     download | |  start download            | | invalidateUserDeviceList
+ *   |   | client        failed | |                            | |
+ *   |   |                      | V                            | |
+ *   |   +------------ DOWNLOAD_IN_PROGRESS -------------------+ |
+ *   |                    |       |                              |
+ *   +<-------------------+       |  download successful         |
+ *   ^                            V                              |
+ *   +----------------------- UP_TO_DATE ------------------------+
+ */
+
 
 // constants for DeviceList._deviceTrackingStatus
-// const TRACKING_STATUS_NOT_TRACKED = 0;
+const TRACKING_STATUS_NOT_TRACKED = 0;
 const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
 const TRACKING_STATUS_DOWNLOAD_IN_PROGRESS = 2;
 const TRACKING_STATUS_UP_TO_DATE = 3;
@@ -36,31 +60,189 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
  * @alias module:crypto/DeviceList
  */
 export default class DeviceList {
-    constructor(baseApis, sessionStore, olmDevice) {
+    constructor(baseApis, cryptoStore, sessionStore, olmDevice) {
+        this._cryptoStore = cryptoStore;
         this._sessionStore = sessionStore;
-        this._serialiser = new DeviceListUpdateSerialiser(
-            baseApis, sessionStore, olmDevice,
-        );
+
+        // userId -> {
+        //     deviceId -> {
+        //         [device info]
+        //     }
+        // }
+        this._devices = {};
 
         // which users we are tracking device status for.
         // userId -> TRACKING_STATUS_*
-        this._deviceTrackingStatus = sessionStore.getEndToEndDeviceTrackingStatus() || {};
+        this._deviceTrackingStatus = {}; // loaded from storage in load()
+
+        // The 'next_batch' sync token at the point the data was writen,
+        // ie. a token representing the point immediately after the
+        // moment represented by the snapshot in the db.
+        this._syncToken = null;
+
+        this._serialiser = new DeviceListUpdateSerialiser(
+            baseApis, olmDevice, this,
+        );
+
+        // userId -> promise
+        this._keyDownloadsInProgressByUser = {};
+
+        // Set whenever changes are made other than setting the sync token
+        this._dirty = false;
+
+        // Promise resolved when device data is saved
+        this._savePromise = null;
+        // Function that resolves the save promise
+        this._resolveSavePromise = null;
+        // The time the save is scheduled for
+        this._savePromiseTime = null;
+        // The timer used to delay the save
+        this._saveTimer = null;
+    }
+
+    /**
+     * Load the device tracking state from storage
+     */
+    async load() {
+        let shouldDeleteSessionStore = false;
+        await this._cryptoStore.doTxn(
+            // migrate from session store if there's data there and not here
+            'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    if (deviceData === null) {
+                        console.log("Migrating e2e device data...");
+                        this._devices = this._sessionStore.getAllEndToEndDevices() || {};
+                        this._deviceTrackingStatus = (
+                            this._sessionStore.getEndToEndDeviceTrackingStatus() || {}
+                        );
+                        this._syncToken = this._sessionStore.getEndToEndDeviceSyncToken();
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                        shouldDeleteSessionStore = true;
+                    } else {
+                        this._devices = deviceData ? deviceData.devices : {},
+                        this._deviceTrackingStatus = deviceData ?
+                            deviceData.trackingStatus : {};
+                        this._syncToken = deviceData ? deviceData.syncToken : null;
+                    }
+                });
+            },
+        );
+
+        if (shouldDeleteSessionStore) {
+            // migrated data is now safely persisted: remove from old store
+            this._sessionStore.removeEndToEndDeviceData();
+        }
+
         for (const u of Object.keys(this._deviceTrackingStatus)) {
             // if a download was in progress when we got shut down, it isn't any more.
             if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
                 this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
             }
         }
-
-        // userId -> promise
-        this._keyDownloadsInProgressByUser = {};
-
-        this.lastKnownSyncToken = null;
     }
 
     /**
-     * Download the keys for a list of users and stores the keys in the session
-     * store.
+     * Save the device tracking state to storage, if any changes are
+     * pending other than updating the sync token
+     *
+     * The actual save will be delayed by a short amount of time to
+     * aggregate multiple writes to the database.
+     *
+     * @param {integer} delay Time in ms before which the save actually happens.
+     *     By default, the save is delayed for a short period in order to batch
+     *     multiple writes, but this behaviour can be disabled by passing 0.
+     *
+     * @return {Promise<bool>} true if the data was saved, false if
+     *     it was not (eg. because no changes were pending). The promise
+     *     will only resolve once the data is saved, so may take some time
+     *     to resolve.
+     */
+    async saveIfDirty(delay) {
+        if (!this._dirty) return Promise.resolve(false);
+        // Delay saves for a bit so we can aggregate multiple saves that happen
+        // in quick succession (eg. when a whole room's devices are marked as known)
+        if (delay === undefined) delay = 500;
+
+        const targetTime = Date.now + delay;
+        if (this._savePromiseTime && targetTime < this._savePromiseTime) {
+            // There's a save scheduled but for after we would like: cancel
+            // it & schedule one for the time we want
+            clearTimeout(this._saveTimer);
+            this._saveTimer = null;
+            this._savePromiseTime = null;
+            // (but keep the save promise since whatever called save before
+            // will still want to know when the save is done)
+        }
+
+        let savePromise = this._savePromise;
+        if (savePromise === null) {
+            savePromise = new Promise((resolve, reject) => {
+                this._resolveSavePromise = resolve;
+            });
+            this._savePromise = savePromise;
+        }
+
+        if (this._saveTimer === null) {
+            const resolveSavePromise = this._resolveSavePromise;
+            this._savePromiseTime = targetTime;
+            this._saveTimer = setTimeout(() => {
+                console.log('Saving device tracking data at token ' + this._syncToken);
+                // null out savePromise now (after the delay but before the write),
+                // otherwise we could return the existing promise when the save has
+                // actually already happened. Likewise for the dirty flag.
+                this._savePromiseTime = null;
+                this._saveTimer = null;
+                this._savePromise = null;
+                this._resolveSavePromise = null;
+
+                this._dirty = false;
+                this._cryptoStore.doTxn(
+                    'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                    },
+                ).then(() => {
+                    resolveSavePromise();
+                });
+            }, delay);
+        }
+        return savePromise;
+    }
+
+    /**
+     * Gets the sync token last set with setSyncToken
+     *
+     * @return {string} The sync token
+     */
+    getSyncToken() {
+        return this._syncToken;
+    }
+
+    /**
+     * Sets the sync token that the app will pass as the 'since' to the /sync
+     * endpoint next time it syncs.
+     * The sync token must always be set after any changes made as a result of
+     * data in that sync since setting the sync token to a newer one will mean
+     * those changed will not be synced from the server if a new client starts
+     * up with that data.
+     *
+     * @param {string} st The sync token
+     */
+    setSyncToken(st) {
+        this._syncToken = st;
+    }
+
+    /**
+     * Ensures up to date keys for a list of users are stored in the session store,
+     * downloading and storing them if they're not (or if forceDownload is
+     * true).
      * @param {Array} userIds The users to fetch.
      * @param {bool} forceDownload Always download the keys even if cached.
      *
@@ -130,7 +312,7 @@ export default class DeviceList {
      * managed to get a list of devices for this user yet.
      */
     getStoredDevicesForUser(userId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
         if (!devs) {
             return null;
         }
@@ -143,6 +325,18 @@ export default class DeviceList {
         return res;
     }
 
+    /**
+     * Get the stored device data for a user, in raw object form
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @return {Object} deviceId->{object} devices, or undefined if
+     * there is no data for this user.
+     */
+    getRawStoredDevicesForUser(userId) {
+        return this._devices[userId];
+    }
+
     /**
      * Get the stored keys for a single device
      *
@@ -153,7 +347,7 @@ export default class DeviceList {
      * if we don't know about this device
      */
     getStoredDevice(userId, deviceId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
         if (!devs || !devs[deviceId]) {
             return undefined;
         }
@@ -178,7 +372,7 @@ export default class DeviceList {
             return null;
         }
 
-        const devices = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devices = this._devices[userId];
         if (!devices) {
             return null;
         }
@@ -207,6 +401,17 @@ export default class DeviceList {
         return null;
     }
 
+    /**
+     * Replaces the list of devices for a user with the given device list
+     *
+     * @param {string} u The user ID
+     * @param {Object} devs New device info for user
+     */
+    storeDevicesForUser(u, devs) {
+        this._devices[u] = devs;
+        this._dirty = true;
+    }
+
     /**
      * flag the given user for device-list tracking, if they are not already.
      *
@@ -232,8 +437,41 @@ export default class DeviceList {
             this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
         }
         // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
+        // of calls; we save all data together once the sync is done
+        this._dirty = true;
+    }
+
+    /**
+     * Mark the given user as no longer being tracked for device-list updates.
+     *
+     * This won't affect any in-progress downloads, which will still go on to
+     * complete; it will just mean that we don't think that we have an up-to-date
+     * list for future calls to downloadKeys.
+     *
+     * @param {String} userId
+     */
+    stopTrackingDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            console.log('No longer tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
+        }
+    }
+
+    /**
+     * Set all users we're currently tracking to untracked
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    stopTrackingAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        this._dirty = true;
     }
 
     /**
@@ -251,21 +489,10 @@ export default class DeviceList {
         if (this._deviceTrackingStatus[userId]) {
             console.log("Marking device list outdated for", userId);
             this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
-        }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
-    }
 
-    /**
-     * Mark all tracked device lists as outdated.
-     *
-     * This will flag each user whose devices we are tracking as in need of an
-     * update.
-     */
-    invalidateAllDeviceLists() {
-        for (const userId of Object.keys(this._deviceTrackingStatus)) {
-            this.invalidateUserDeviceList(userId);
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
         }
     }
 
@@ -276,6 +503,8 @@ export default class DeviceList {
      *    is no need to wait for this (it's mostly for the unit tests).
      */
     refreshOutdatedDeviceLists() {
+        this.saveIfDirty();
+
         const usersToDownload = [];
         for (const userId of Object.keys(this._deviceTrackingStatus)) {
             const stat = this._deviceTrackingStatus[userId];
@@ -283,17 +512,21 @@ export default class DeviceList {
                 usersToDownload.push(userId);
             }
         }
-        if (usersToDownload.length == 0) {
-            return;
-        }
-
-        // we didn't persist the tracking status during
-        // invalidateUserDeviceList, so do it now.
-        this._persistDeviceTrackingStatus();
 
         return this._doKeyDownload(usersToDownload);
     }
 
+    /**
+     * Set the stored device data for a user, in raw object form
+     * Used only by internal class DeviceListUpdateSerialiser
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @param {Object} devices deviceId->{object} the new devices
+     */
+    _setRawStoredDevicesForUser(userId, devices) {
+        this._devices[userId] = devices;
+    }
 
     /**
      * Fire off download update requests for the given users, and update the
@@ -313,7 +546,7 @@ export default class DeviceList {
         }
 
         const prom = this._serialiser.updateDevicesForUsers(
-            users, this.lastKnownSyncToken,
+            users, this._syncToken,
         ).then(() => {
             finished(true);
         }, (e) => {
@@ -334,6 +567,8 @@ export default class DeviceList {
 
         const finished = (success) => {
             users.forEach((u) => {
+                this._dirty = true;
+
                 // we may have queued up another download request for this user
                 // since we started this request. If that happens, we should
                 // ignore the completion of the first one.
@@ -355,15 +590,11 @@ export default class DeviceList {
                     }
                 }
             });
-            this._persistDeviceTrackingStatus();
+            this.saveIfDirty();
         };
 
         return prom;
     }
-
-    _persistDeviceTrackingStatus() {
-        this._sessionStore.storeEndToEndDeviceTrackingStatus(this._deviceTrackingStatus);
-    }
 }
 
 /**
@@ -376,10 +607,15 @@ export default class DeviceList {
  * time (and queuing other requests up).
  */
 class DeviceListUpdateSerialiser {
-    constructor(baseApis, sessionStore, olmDevice) {
+    /*
+     * @param {object} baseApis Base API object
+     * @param {object} olmDevice The Olm Device
+     * @param {object} deviceList The device list object
+     */
+    constructor(baseApis, olmDevice, deviceList) {
         this._baseApis = baseApis;
-        this._sessionStore = sessionStore;
         this._olmDevice = olmDevice;
+        this._deviceList = deviceList; // the device list to be updated
 
         this._downloadInProgress = false;
 
@@ -392,9 +628,7 @@ class DeviceListUpdateSerialiser {
         // non-null indicates that we have users queued for download.
         this._queuedQueryDeferred = null;
 
-        // sync token to be used for the next query: essentially the
-        // most recent one we know about
-        this._nextSyncToken = null;
+        this._syncToken = null; // The sync token we send with the requests
     }
 
     /**
@@ -413,12 +647,16 @@ class DeviceListUpdateSerialiser {
         users.forEach((u) => {
             this._keyDownloadsQueuedByUser[u] = true;
         });
-        this._nextSyncToken = syncToken;
 
         if (!this._queuedQueryDeferred) {
             this._queuedQueryDeferred = Promise.defer();
         }
 
+        // We always take the new sync token and just use the latest one we've
+        // been given, since it just needs to be at least as recent as the
+        // sync response the device invalidation message arrived in
+        this._syncToken = syncToken;
+
         if (this._downloadInProgress) {
             // just queue up these users
             console.log('Queued key download for', users);
@@ -445,8 +683,8 @@ class DeviceListUpdateSerialiser {
         this._downloadInProgress = true;
 
         const opts = {};
-        if (this._nextSyncToken) {
-            opts.token = this._nextSyncToken;
+        if (this._syncToken) {
+            opts.token = this._syncToken;
         }
 
         this._baseApis.downloadKeysForUsers(
@@ -479,7 +717,7 @@ class DeviceListUpdateSerialiser {
             }
         }, (e) => {
             console.warn('Error downloading keys for ' + downloadUsers + ':', e);
-            this._downloadInProgressInProgress = false;
+            this._downloadInProgress = false;
             deferred.reject(e);
         });
 
@@ -491,7 +729,7 @@ class DeviceListUpdateSerialiser {
 
         // map from deviceid -> deviceinfo for this user
         const userStore = {};
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
         if (devs) {
             Object.keys(devs).forEach((deviceId) => {
                 const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
@@ -503,15 +741,13 @@ class DeviceListUpdateSerialiser {
             this._olmDevice, userId, userStore, response || {},
         );
 
-        // update the session store
+        // put the updates into thr object that will be returned as our results
         const storage = {};
         Object.keys(userStore).forEach((deviceId) => {
             storage[deviceId] = userStore[deviceId].toStorage();
         });
 
-        this._sessionStore.storeEndToEndDevicesForUser(
-            userId, storage,
-        );
+        this._deviceList._setRawStoredDevicesForUser(userId, storage);
     }
 }
 

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -35,13 +35,19 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * The state machine looks like:
  *
- *     |
- *     V         (cancellation requested)
- *   UNSENT  -----------------------------+
- *     |                                  |
- *     | (send successful)                |
- *     V                                  |
- *    SENT                                |
+ *     |         (cancellation sent)
+ *     | .-------------------------------------------------.
+ *     | |                                                 |
+ *     V V       (cancellation requested)                  |
+ *   UNSENT  -----------------------------+                |
+ *     |                                  |                |
+ *     |                                  |                |
+ *     | (send successful)                |  CANCELLATION_PENDING_AND_WILL_RESEND
+ *     V                                  |                Λ
+ *    SENT                                |                |
+ *     |--------------------------------  |  --------------'
+ *     |                                  |  (cancellation requested with intent
+ *     |                                  |   to resend the original request)
  *     |                                  |
  *     | (cancellation requested)         |
  *     V                                  |
@@ -62,6 +68,12 @@ const ROOM_KEY_REQUEST_STATES = {
 
     /** reply received, cancellation not yet sent */
     CANCELLATION_PENDING: 2,
+
+    /**
+     * Cancellation not yet sent and will transition to UNSENT instead of
+     * being deleted once the cancellation has been sent.
+     */
+    CANCELLATION_PENDING_AND_WILL_RESEND: 3,
 };
 
 export default class OutgoingRoomKeyRequestManager {
@@ -130,14 +142,16 @@ export default class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Cancel room key requests, if any match the given details
+     * Cancel room key requests, if any match the given requestBody
      *
      * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {boolean} andResend if true, transition to UNSENT instead of
+     *                            deleting after sending cancellation.
      *
      * @returns {Promise} resolves when the request has been updated in our
      *    pending list.
      */
-    cancelRoomKeyRequest(requestBody) {
+    cancelRoomKeyRequest(requestBody, andResend=false) {
         return this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         ).then((req) => {
@@ -147,6 +161,7 @@ export default class OutgoingRoomKeyRequestManager {
             }
             switch (req.state) {
                 case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
                     // nothing to do here
                     return;
 
@@ -166,11 +181,16 @@ export default class OutgoingRoomKeyRequestManager {
                         req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
                     );
 
-                case ROOM_KEY_REQUEST_STATES.SENT:
+                case ROOM_KEY_REQUEST_STATES.SENT: {
+                    // If `andResend` is set, transition to UNSENT once the
+                    // cancellation has successfully been sent.
+                    const state = andResend ?
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING;
                     // send a cancellation.
                     return this._cryptoStore.updateOutgoingRoomKeyRequest(
                         req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
-                            state: ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+                            state,
                             cancellationTxnId: this._baseApis.makeTxnId(),
                         },
                     ).then((updatedReq) => {
@@ -200,15 +220,23 @@ export default class OutgoingRoomKeyRequestManager {
                         // do.)
                         this._sendOutgoingRoomKeyRequestCancellation(
                             updatedReq,
+                            andResend,
                         ).catch((e) => {
                             console.error(
                                 "Error sending room key request cancellation;"
                                 + " will retry later.", e,
                             );
                             this._startTimer();
-                        }).done();
+                        }).then(() => {
+                            if (!andResend) return;
+                            // The request has transitioned from
+                            // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
+                            // still need to resend the request which is now UNSENT, so
+                            // start the timer if it isn't already started.
+                            this._startTimer();
+                        });
                     });
-
+                }
                 default:
                     throw new Error('unhandled state: ' + req.state);
             }
@@ -258,6 +286,7 @@ export default class OutgoingRoomKeyRequestManager {
 
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
@@ -267,10 +296,16 @@ export default class OutgoingRoomKeyRequestManager {
             }
 
             let prom;
-            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
-                prom = this._sendOutgoingRoomKeyRequest(req);
-            } else { // must be a cancellation
-                prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    prom = this._sendOutgoingRoomKeyRequest(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req, true);
+                    break;
             }
 
             return prom.then(() => {
@@ -309,8 +344,9 @@ export default class OutgoingRoomKeyRequestManager {
         });
     }
 
-    // given a RoomKeyRequest, cancel it and delete the request record
-    _sendOutgoingRoomKeyRequestCancellation(req) {
+    // Given a RoomKeyRequest, cancel it and delete the request record unless
+    // andResend is set, in which case transition to UNSENT.
+    _sendOutgoingRoomKeyRequestCancellation(req, andResend) {
         console.log(
             `Sending cancellation for key request for ` +
             `${stringifyRequestBody(req.requestBody)} to ` +
@@ -327,6 +363,14 @@ export default class OutgoingRoomKeyRequestManager {
         return this._sendMessageToDevices(
             requestMessage, req.recipients, req.cancellationTxnId,
         ).then(() => {
+            if (andResend) {
+                // We want to resend, so transition to UNSENT
+                return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId,
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+                    { state: ROOM_KEY_REQUEST_STATES.UNSENT },
+                );
+            }
             return this._cryptoStore.deleteOutgoingRoomKeyRequest(
                 req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             );

src/crypto/RoomList.js

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module crypto/RoomList
+ *
+ * Manages the list of encrypted rooms
+ */
+
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+/**
+ * @alias module:crypto/RoomList
+ */
+export default class RoomList {
+    constructor(cryptoStore, sessionStore) {
+        this._cryptoStore = cryptoStore;
+        this._sessionStore = sessionStore;
+
+        // Object of roomId -> room e2e info object (body of the m.room.encryption event)
+        this._roomEncryption = {};
+    }
+
+    async init() {
+        let removeSessionStoreRooms = false;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.getEndToEndRooms(txn, (result) => {
+                    if (result === null || Object.keys(result).length === 0) {
+                        // migrate from session store, if there's data there
+                        const sessStoreRooms = this._sessionStore.getAllEndToEndRooms();
+                        if (sessStoreRooms !== null) {
+                            for (const roomId of Object.keys(sessStoreRooms)) {
+                                this._cryptoStore.storeEndToEndRoom(
+                                    roomId, sessStoreRooms[roomId], txn,
+                                );
+                            }
+                        }
+                        this._roomEncryption = sessStoreRooms;
+                        removeSessionStoreRooms = true;
+                    } else {
+                        this._roomEncryption = result;
+                    }
+                });
+            },
+        );
+        if (removeSessionStoreRooms) {
+            this._sessionStore.removeAllEndToEndRooms();
+        }
+    }
+
+    getRoomEncryption(roomId) {
+        return this._roomEncryption[roomId] || null;
+    }
+
+    isRoomEncrypted(roomId) {
+        return Boolean(this.getRoomEncryption(roomId));
+    }
+
+    async setRoomEncryption(roomId, roomInfo) {
+        this._roomEncryption[roomId] = roomInfo;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+            },
+        );
+    }
+}

src/crypto/algorithms/base.js

@@ -117,7 +117,8 @@ class DecryptionAlgorithm {
      *
      * @param {MatrixEvent} event undecrypted event
      *
-     * @return {Promise} resolves once we have finished decrypting. Rejects with an
+     * @return {Promise<module:crypto~EventDecryptionResult>} promise which
+     * resolves once we have finished decrypting. Rejects with an
      * `algorithms.DecryptionError` if there is a problem decrypting the event.
      */
 
@@ -178,31 +179,26 @@ class DecryptionError extends Error {
     constructor(msg, details) {
         super(msg);
         this.name = 'DecryptionError';
-        this.details = details;
-    }
-
-    /**
-     * override the string used when logging
-     *
-     * @returns {String}
-     */
-    toString() {
-        let result = this.name + '[msg: ' + this.message;
-
-        if (this.details) {
-            result += ', ' +
-                Object.keys(this.details).map(
-                    (k) => k + ': ' + this.details[k],
-                ).join(', ');
-        }
-
-        result += ']';
-
-        return result;
+        this.detailedString = _detailedStringForDecryptionError(this, details);
     }
 }
 export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
+function _detailedStringForDecryptionError(err, details) {
+    let result = err.name + '[msg: ' + err.message;
+
+    if (details) {
+        result += ', ' +
+            Object.keys(details).map(
+                (k) => k + ': ' + details[k],
+            ).join(', ');
+    }
+
+    result += ']';
+
+    return result;
+}
+
 /**
  * Exception thrown specifically when we want to warn the user to consider
  * the security of their conversation before continuing

src/crypto/algorithms/megolm.js

@@ -74,6 +74,14 @@ OutboundSessionInfo.prototype.needsRotation = function(
     return false;
 };
 
+OutboundSessionInfo.prototype.markSharedWithDevice = function(
+    userId, deviceId, chainIndex,
+) {
+    if (!this.sharedWithDevices[userId]) {
+        this.sharedWithDevices[userId] = {};
+    }
+    this.sharedWithDevices[userId][deviceId] = chainIndex;
+};
 
 /**
  * Determine if this session has been shared with devices which it shouldn't
@@ -249,7 +257,7 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
     const sessionId = this._olmDevice.createOutboundGroupSession();
     const key = this._olmDevice.getOutboundGroupSessionKey(sessionId);
 
-    this._olmDevice.addInboundGroupSession(
+    await this._olmDevice.addInboundGroupSession(
         this._roomId, this._olmDevice.deviceCurve25519Key, [], sessionId,
         key.key, {ed25519: this._olmDevice.deviceEd25519Key},
     );
@@ -262,15 +270,151 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
  *
  * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
  *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceId>} devicemap
+ *   mapping from userId to deviceId to {@link module:crypto~OlmSessionResult}
+ *
  * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
  *    map from userid to list of devices
  *
- * @return {module:client.Promise} Promise which resolves once the key sharing
- *     message has been sent.
+ * @return {array<object<userid, deviceInfo>>}
  */
-MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUser) {
-    const self = this;
+MegolmEncryption.prototype._splitUserDeviceMap = function(
+    session, chainIndex, devicemap, devicesByUser,
+) {
+    const maxToDeviceMessagesPerRequest = 20;
 
+    // use an array where the slices of a content map gets stored
+    const mapSlices = [];
+    let currentSliceId = 0; // start inserting in the first slice
+    let entriesInCurrentSlice = 0;
+
+    for (const userId of Object.keys(devicesByUser)) {
+        const devicesToShareWith = devicesByUser[userId];
+        const sessionResults = devicemap[userId];
+
+        for (let i = 0; i < devicesToShareWith.length; i++) {
+            const deviceInfo = devicesToShareWith[i];
+            const deviceId = deviceInfo.deviceId;
+
+            const sessionResult = sessionResults[deviceId];
+            if (!sessionResult.sessionId) {
+                // no session with this device, probably because there
+                // were no one-time keys.
+                //
+                // we could send them a to_device message anyway, as a
+                // signal that they have missed out on the key sharing
+                // message because of the lack of keys, but there's not
+                // much point in that really; it will mostly serve to clog
+                // up to_device inboxes.
+
+                // mark this device as "handled" because we don't want to try
+                // to claim a one-time-key for dead devices on every message.
+                session.markSharedWithDevice(userId, deviceId, chainIndex);
+
+                // ensureOlmSessionsForUsers has already done the logging,
+                // so just skip it.
+                continue;
+            }
+
+            console.log(
+                "share keys with device " + userId + ":" + deviceId,
+            );
+
+            if (entriesInCurrentSlice > maxToDeviceMessagesPerRequest) {
+                // the current slice is filled up. Start inserting into the next slice
+                entriesInCurrentSlice = 0;
+                currentSliceId++;
+            }
+            if (!mapSlices[currentSliceId]) {
+                mapSlices[currentSliceId] = [];
+            }
+
+            mapSlices[currentSliceId].push({
+                userId: userId,
+                deviceInfo: deviceInfo,
+            });
+
+            entriesInCurrentSlice++;
+        }
+    }
+    return mapSlices;
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceInfo>} userDeviceMap
+ *   mapping from userId to deviceInfo
+ *
+ * @param {object} payload fields to include in the encrypted payload
+ *
+ * @return {module:client.Promise} Promise which resolves once the key sharing
+ *     for the given userDeviceMap is generated and has been sent.
+ */
+MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
+    session, chainIndex, userDeviceMap, payload,
+) {
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: this._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };
+    const contentMap = {};
+
+    const promises = [];
+    for (let i = 0; i < userDeviceMap.length; i++) {
+        const val = userDeviceMap[i];
+        const userId = val.userId;
+        const deviceInfo = val.deviceInfo;
+        const deviceId = deviceInfo.deviceId;
+
+        if (!contentMap[userId]) {
+            contentMap[userId] = {};
+        }
+        contentMap[userId][deviceId] = encryptedContent;
+
+        promises.push(
+            olmlib.encryptMessageForDevice(
+                encryptedContent.ciphertext,
+                this._userId,
+                this._deviceId,
+                this._olmDevice,
+                userId,
+                deviceInfo,
+                payload,
+            ),
+        );
+    }
+
+    return Promise.all(promises).then(() => {
+        return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
+            // store that we successfully uploaded the keys of the current slice
+            for (const userId of Object.keys(contentMap)) {
+                for (const deviceId of Object.keys(contentMap[userId])) {
+                    session.markSharedWithDevice(
+                        userId, deviceId, chainIndex,
+                    );
+                }
+            }
+        });
+    });
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices
+ */
+MegolmEncryption.prototype._shareKeyWithDevices = async function(session, devicesByUser) {
     const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
     const payload = {
         type: "m.room_key",
@@ -283,104 +427,28 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
         },
     };
 
-    const contentMap = {};
-
-    return olmlib.ensureOlmSessionsForDevices(
+    const devicemap = await olmlib.ensureOlmSessionsForDevices(
         this._olmDevice, this._baseApis, devicesByUser,
-    ).then(function(devicemap) {
-        const promises = [];
+    );
 
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
+    const userDeviceMaps = this._splitUserDeviceMap(
+        session, key.chain_index, devicemap, devicesByUser,
+    );
 
-            const devicesToShareWith = devicesByUser[userId];
-            const sessionResults = devicemap[userId];
+    for (let i = 0; i < userDeviceMaps.length; i++) {
+        try {
+            await this._encryptAndSendKeysToDevices(
+                session, key.chain_index, userDeviceMaps[i], payload,
+            );
+            console.log(`Completed megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length})`);
+        } catch (e) {
+            console.log(`megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length}) failed`);
 
-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                const deviceId = deviceInfo.deviceId;
-
-                const sessionResult = sessionResults[deviceId];
-                if (!sessionResult.sessionId) {
-                    // no session with this device, probably because there
-                    // were no one-time keys.
-                    //
-                    // we could send them a to_device message anyway, as a
-                    // signal that they have missed out on the key sharing
-                    // message because of the lack of keys, but there's not
-                    // much point in that really; it will mostly serve to clog
-                    // up to_device inboxes.
-                    //
-                    // ensureOlmSessionsForUsers has already done the logging,
-                    // so just skip it.
-                    continue;
-                }
-
-                console.log(
-                    "sharing keys with device " + userId + ":" + deviceId,
-                );
-
-                const encryptedContent = {
-                    algorithm: olmlib.OLM_ALGORITHM,
-                    sender_key: self._olmDevice.deviceCurve25519Key,
-                    ciphertext: {},
-                };
-
-                if (!contentMap[userId]) {
-                    contentMap[userId] = {};
-                }
-
-                contentMap[userId][deviceId] = encryptedContent;
-
-                promises.push(
-                    olmlib.encryptMessageForDevice(
-                        encryptedContent.ciphertext,
-                        self._userId,
-                        self._deviceId,
-                        self._olmDevice,
-                        userId,
-                        deviceInfo,
-                        payload,
-                    ),
-                );
-            }
+            throw e;
         }
-
-        if (promises.length === 0) {
-            // no devices to send to
-            return Promise.resolve();
-        }
-
-        return Promise.all(promises).then(() => {
-            // TODO: retries
-            return self._baseApis.sendToDevice("m.room.encrypted", contentMap);
-        });
-    }).then(function() {
-        console.log(`Completed megolm keyshare in ${self._roomId}`);
-
-        // Add the devices we have shared with to session.sharedWithDevices.
-        //
-        // we deliberately iterate over devicesByUser (ie, the devices we
-        // attempted to share with) rather than the contentMap (those we did
-        // share with), because we don't want to try to claim a one-time-key
-        // for dead devices on every message.
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
-            if (!session.sharedWithDevices[userId]) {
-                session.sharedWithDevices[userId] = {};
-            }
-            const devicesToShareWith = devicesByUser[userId];
-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                session.sharedWithDevices[userId][deviceInfo.deviceId] =
-                    key.chain_index;
-            }
-        }
-    });
+    }
 };
 
 /**
@@ -473,6 +541,12 @@ MegolmEncryption.prototype._getDevicesInRoom = function(room) {
         return u.userId;
     });
 
+    // The global value is treated as a default for when rooms don't specify a value.
+    let isBlacklisting = this._crypto.getGlobalBlacklistUnverifiedDevices();
+    if (typeof room.getBlacklistUnverifiedDevices() === 'boolean') {
+        isBlacklisting = room.getBlacklistUnverifiedDevices();
+    }
+
     // We are happy to use a cached version here: we assume that if we already
     // have a list of the user's devices, then we already share an e2e room
     // with them, which means that they will have announced any new devices via
@@ -496,9 +570,7 @@ MegolmEncryption.prototype._getDevicesInRoom = function(room) {
                 }
 
                 if (userDevices[deviceId].isBlocked() ||
-                    (userDevices[deviceId].isUnverified() &&
-                     (room.getBlacklistUnverifiedDevices() ||
-                      this._crypto.getGlobalBlacklistUnverifiedDevices()))
+                    (userDevices[deviceId].isUnverified() && isBlacklisting)
                    ) {
                     delete userDevices[deviceId];
                 }
@@ -522,7 +594,7 @@ function MegolmDecryption(params) {
     base.DecryptionAlgorithm.call(this, params);
 
     // events which we couldn't decrypt due to unknown sessions / indexes: map from
-    // senderKey|sessionId to list of MatrixEvents
+    // senderKey|sessionId to Set of MatrixEvents
     this._pendingEvents = {};
 
     // this gets stubbed out by the unit tests.
@@ -535,18 +607,12 @@ utils.inherits(MegolmDecryption, base.DecryptionAlgorithm);
  *
  * @param {MatrixEvent} event
  *
- * @return {Promise} resolves once we have finished decrypting. Rejects with an
- * `algorithms.DecryptionError` if there is a problem decrypting the event.
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting, or rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
  */
-MegolmDecryption.prototype.decryptEvent = function(event) {
-    return this._decryptEvent(event, true);
-};
-
-
-// helper for the real decryptEvent and for _retryDecryption. If
-// requestKeysOnFail is true, we'll send an m.room_key_request when we fail
-// to decrypt the event due to missing megolm keys.
-MegolmDecryption.prototype._decryptEvent = async function(event, requestKeysOnFail) {
+MegolmDecryption.prototype.decryptEvent = async function(event) {
     const content = event.getWireContent();
 
     if (!content.sender_key || !content.session_id ||
@@ -555,17 +621,22 @@ MegolmDecryption.prototype._decryptEvent = async function(event, requestKeysOnFa
         throw new base.DecryptionError("Missing fields in input");
     }
 
+    // we add the event to the pending list *before* we start decryption.
+    //
+    // then, if the key turns up while decryption is in progress (and
+    // decryption fails), we will schedule a retry.
+    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    this._addEventToPendingList(event);
+
     let res;
     try {
-        res = this._olmDevice.decryptGroupMessage(
+        res = await this._olmDevice.decryptGroupMessage(
             event.getRoomId(), content.sender_key, content.session_id, content.ciphertext,
+            event.getId(), event.getTs(),
         );
     } catch (e) {
         if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
-            this._addEventToPendingList(event);
-            if (requestKeysOnFail) {
-                this._requestKeysForEvent(event);
-            }
+            this._requestKeysForEvent(event);
         }
         throw new base.DecryptionError(
             e.toString(), {
@@ -576,10 +647,13 @@ MegolmDecryption.prototype._decryptEvent = async function(event, requestKeysOnFa
 
     if (res === null) {
         // We've got a message for a session we don't have.
-        this._addEventToPendingList(event);
-        if (requestKeysOnFail) {
-            this._requestKeysForEvent(event);
-        }
+        //
+        // (XXX: We might actually have received this key since we started
+        // decrypting, in which case we'll have scheduled a retry, and this
+        // request will be redundant. We could probably check to see if the
+        // event is still in the pending list; if not, a retry will have been
+        // scheduled, so we needn't send out the request here.)
+        this._requestKeysForEvent(event);
         throw new base.DecryptionError(
             "The sender's device has not sent us the keys for this message.",
             {
@@ -588,6 +662,10 @@ MegolmDecryption.prototype._decryptEvent = async function(event, requestKeysOnFa
         );
     }
 
+    // success. We can remove the event from the pending list, if that hasn't
+    // already happened.
+    this._removeEventFromPendingList(event);
+
     const payload = JSON.parse(res.result);
 
     // belt-and-braces check that the room id matches that indicated by the HS
@@ -599,8 +677,12 @@ MegolmDecryption.prototype._decryptEvent = async function(event, requestKeysOnFa
         );
     }
 
-    event.setClearData(payload, res.senderKey, res.keysClaimed.ed25519,
-        res.forwardingCurve25519KeyChain);
+    return {
+        clearEvent: payload,
+        senderCurve25519Key: res.senderKey,
+        claimedEd25519Key: res.keysClaimed.ed25519,
+        forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+    };
 };
 
 MegolmDecryption.prototype._requestKeysForEvent = function(event) {
@@ -627,8 +709,7 @@ MegolmDecryption.prototype._requestKeysForEvent = function(event) {
 };
 
 /**
- * Add an event to the list of those we couldn't decrypt the first time we
- * saw them.
+ * Add an event to the list of those awaiting their session keys.
  *
  * @private
  *
@@ -638,11 +719,32 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
     const content = event.getWireContent();
     const k = content.sender_key + "|" + content.session_id;
     if (!this._pendingEvents[k]) {
-        this._pendingEvents[k] = [];
+        this._pendingEvents[k] = new Set();
     }
-    this._pendingEvents[k].push(event);
+    this._pendingEvents[k].add(event);
 };
 
+/**
+ * Remove an event from the list of those awaiting their session keys.
+ *
+ * @private
+ *
+ * @param {module:models/event.MatrixEvent} event
+ */
+MegolmDecryption.prototype._removeEventFromPendingList = function(event) {
+    const content = event.getWireContent();
+    const k = content.sender_key + "|" + content.session_id;
+    if (!this._pendingEvents[k]) {
+        return;
+    }
+
+    this._pendingEvents[k].delete(event);
+    if (this._pendingEvents[k].size === 0) {
+        delete this._pendingEvents[k];
+    }
+};
+
+
 /**
  * @inheritdoc
  *
@@ -706,24 +808,26 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
         content.room_id, senderKey, forwardingKeyChain, sessionId,
         content.session_key, keysClaimed,
         exportFormat,
-    );
+    ).then(() => {
+        // cancel any outstanding room key requests for this session
+        this._crypto.cancelRoomKeyRequest({
+            algorithm: content.algorithm,
+            room_id: content.room_id,
+            session_id: content.session_id,
+            sender_key: senderKey,
+        });
 
-    // cancel any outstanding room key requests for this session
-    this._crypto.cancelRoomKeyRequest({
-        algorithm: content.algorithm,
-        room_id: content.room_id,
-        session_id: content.session_id,
-        sender_key: senderKey,
+        // have another go at decrypting events sent with this session.
+        this._retryDecryption(senderKey, sessionId);
+    }).catch((e) => {
+        console.error(`Error handling m.room_key_event: ${e}`);
     });
-
-    // have another go at decrypting events sent with this session.
-    this._retryDecryption(senderKey, sessionId);
 };
 
 /**
  * @inheritdoc
  */
-MegolmDecryption.prototype.hasKeysForKeyRequest = async function(keyRequest) {
+MegolmDecryption.prototype.hasKeysForKeyRequest = function(keyRequest) {
     const body = keyRequest.requestBody;
 
     return this._olmDevice.hasInboundSessionKeys(
@@ -764,10 +868,10 @@ MegolmDecryption.prototype.shareKeysWithDevice = function(keyRequest) {
             + userId + ":" + deviceId,
         );
 
-        const payload = this._buildKeyForwardingMessage(
+        return this._buildKeyForwardingMessage(
             body.room_id, body.sender_key, body.session_id,
         );
-
+    }).then((payload) => {
         const encryptedContent = {
             algorithm: olmlib.OLM_ALGORITHM,
             sender_key: this._olmDevice.deviceCurve25519Key,
@@ -795,10 +899,10 @@ MegolmDecryption.prototype.shareKeysWithDevice = function(keyRequest) {
     }).done();
 };
 
-MegolmDecryption.prototype._buildKeyForwardingMessage = function(
+MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
     roomId, senderKey, sessionId,
 ) {
-    const key = this._olmDevice.getInboundGroupSessionKey(
+    const key = await this._olmDevice.getInboundGroupSessionKey(
         roomId, senderKey, sessionId,
     );
 
@@ -823,10 +927,18 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = function(
  * @param {module:crypto/OlmDevice.MegolmSessionData} session
  */
 MegolmDecryption.prototype.importRoomKey = function(session) {
-    this._olmDevice.importInboundGroupSession(session);
-
-    // have another go at decrypting events sent with this session.
-    this._retryDecryption(session.sender_key, session.session_id);
+    return this._olmDevice.addInboundGroupSession(
+        session.room_id,
+        session.sender_key,
+        session.forwarding_curve25519_key_chain,
+        session.session_id,
+        session.session_key,
+        session.sender_claimed_keys,
+        true,
+    ).then(() => {
+        // have another go at decrypting events sent with this session.
+        this._retryDecryption(session.sender_key, session.session_id);
+    });
 };
 
 /**
@@ -845,8 +957,8 @@ MegolmDecryption.prototype._retryDecryption = function(senderKey, sessionId) {
 
     delete this._pendingEvents[k];
 
-    for (let i = 0; i < pending.length; i++) {
-        pending[i].attemptDecryption(this._crypto);
+    for (const ev of pending) {
+        ev.attemptDecryption(this._crypto);
     }
 };
 

src/crypto/algorithms/olm.js

@@ -157,8 +157,10 @@ utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
  *
  * @param {MatrixEvent} event
  *
- * returns a promise which resolves once we have finished decrypting. Rejects with an
- * `algorithms.DecryptionError` if there is a problem decrypting the event.
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting. Rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
  */
 OlmDecryption.prototype.decryptEvent = async function(event) {
     const content = event.getWireContent();
@@ -227,9 +229,13 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
     }
 
     const claimedKeys = payload.keys || {};
-    event.setClearData(payload, deviceKey, claimedKeys.ed25519 || null);
-};
 
+    return {
+        clearEvent: payload,
+        senderCurve25519Key: deviceKey,
+        claimedEd25519Key: claimedKeys.ed25519 || null,
+    };
+};
 
 /**
  * Attempt to decrypt an Olm message
@@ -242,14 +248,16 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
 ) {
-    const sessionIds = this._olmDevice.getSessionIdsForDevice(theirDeviceIdentityKey);
+    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
+        theirDeviceIdentityKey,
+    );
 
     // try each session in turn.
     const decryptionErrors = {};
     for (let i = 0; i < sessionIds.length; i++) {
         const sessionId = sessionIds[i];
         try {
-            const payload = this._olmDevice.decryptMessage(
+            const payload = await this._olmDevice.decryptMessage(
                 theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
             console.log(
@@ -258,7 +266,7 @@ OlmDecryption.prototype._decryptMessage = async function(
             );
             return payload;
         } catch (e) {
-            const foundSession = this._olmDevice.matchesSession(
+            const foundSession = await this._olmDevice.matchesSession(
                 theirDeviceIdentityKey, sessionId, message.type, message.body,
             );
 
@@ -296,7 +304,7 @@ OlmDecryption.prototype._decryptMessage = async function(
 
     let res;
     try {
-        res = this._olmDevice.createInboundSession(
+        res = await this._olmDevice.createInboundSession(
             theirDeviceIdentityKey, message.type, message.body,
         );
     } catch (e) {

src/crypto/index.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,6 +34,7 @@ const DeviceVerification = DeviceInfo.DeviceVerification;
 const DeviceList = require('./DeviceList').default;
 
 import OutgoingRoomKeyRequestManager from './OutgoingRoomKeyRequestManager';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
 
 /**
  * Cryptography bits
@@ -57,19 +59,23 @@ import OutgoingRoomKeyRequestManager from './OutgoingRoomKeyRequestManager';
  *
  * @param {module:crypto/store/base~CryptoStore} cryptoStore
  *    storage for the crypto layer.
+ *
+ * @param {RoomList} roomList An initialised RoomList object
  */
 function Crypto(baseApis, sessionStore, userId, deviceId,
-                clientStore, cryptoStore) {
+                clientStore, cryptoStore, roomList) {
     this._baseApis = baseApis;
     this._sessionStore = sessionStore;
     this._userId = userId;
     this._deviceId = deviceId;
     this._clientStore = clientStore;
     this._cryptoStore = cryptoStore;
+    this._roomList = roomList;
 
-    this._olmDevice = new OlmDevice(sessionStore);
-    this._deviceList = new DeviceList(baseApis, sessionStore, this._olmDevice);
-    this._initialDeviceListInvalidationPending = false;
+    this._olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    this._deviceList = new DeviceList(
+        baseApis, cryptoStore, sessionStore, this._olmDevice,
+    );
 
     // the last time we did a check for the number of one-time-keys on the
     // server.
@@ -88,12 +94,6 @@ function Crypto(baseApis, sessionStore, userId, deviceId,
 
     this._deviceKeys = {};
 
-    // build our device keys: these will later be uploaded
-    this._deviceKeys["ed25519:" + this._deviceId] =
-        this._olmDevice.deviceEd25519Key;
-    this._deviceKeys["curve25519:" + this._deviceId] =
-        this._olmDevice.deviceCurve25519Key;
-
     this._globalBlacklistUnverifiedDevices = false;
 
     this._outgoingRoomKeyRequestManager = new OutgoingRoomKeyRequestManager(
@@ -106,8 +106,43 @@ function Crypto(baseApis, sessionStore, userId, deviceId,
     this._receivedRoomKeyRequestCancellations = [];
     // true if we are currently processing received room key requests
     this._processingRoomKeyRequests = false;
+}
+utils.inherits(Crypto, EventEmitter);
 
-    let myDevices = this._sessionStore.getEndToEndDevicesForUser(
+/**
+ * Initialise the crypto module so that it is ready for use
+ *
+ * Returns a promise which resolves once the crypto module is ready for use.
+ */
+Crypto.prototype.init = async function() {
+    const sessionStoreHasAccount = Boolean(this._sessionStore.getEndToEndAccount());
+    let cryptoStoreHasAccount;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
+            this._cryptoStore.getAccount(txn, (pickledAccount) => {
+                cryptoStoreHasAccount = Boolean(pickledAccount);
+            });
+        },
+    );
+    if (sessionStoreHasAccount && !cryptoStoreHasAccount) {
+        // we're about to migrate to the crypto store
+        this.emit("crypto.warning", 'CRYPTO_WARNING_ACCOUNT_MIGRATED');
+    } else if (sessionStoreHasAccount && cryptoStoreHasAccount) {
+        // There's an account in both stores: an old version of
+        // the code has been run against this store.
+        this.emit("crypto.warning", 'CRYPTO_WARNING_OLD_VERSION_DETECTED');
+    }
+
+    await this._olmDevice.init();
+    await this._deviceList.load();
+
+    // build our device keys: these will later be uploaded
+    this._deviceKeys["ed25519:" + this._deviceId] =
+        this._olmDevice.deviceEd25519Key;
+    this._deviceKeys["curve25519:" + this._deviceId] =
+        this._olmDevice.deviceCurve25519Key;
+
+    let myDevices = this._deviceList.getRawStoredDevicesForUser(
         this._userId,
     );
 
@@ -125,17 +160,11 @@ function Crypto(baseApis, sessionStore, userId, deviceId,
         };
 
         myDevices[this._deviceId] = deviceInfo;
-        this._sessionStore.storeEndToEndDevicesForUser(
+        this._deviceList.storeDevicesForUser(
             this._userId, myDevices,
         );
+        this._deviceList.saveIfDirty();
     }
-}
-utils.inherits(Crypto, EventEmitter);
-
-/**
- * Initialise the crypto module so that it is ready for use
- */
-Crypto.prototype.init = async function() {
 };
 
 /**
@@ -147,15 +176,6 @@ Crypto.prototype.init = async function() {
  */
 Crypto.prototype.registerEventHandlers = function(eventEmitter) {
     const crypto = this;
-    eventEmitter.on("sync", function(syncState, oldState, data) {
-        try {
-            if (syncState === "SYNCING") {
-                crypto._onSyncCompleted(data);
-            }
-        } catch (e) {
-            console.error("Error handling sync", e);
-        }
-    });
 
     eventEmitter.on("RoomMember.membership", function(event, member, oldMembership) {
         try {
@@ -199,18 +219,17 @@ Crypto.prototype.getDeviceEd25519Key = function() {
 
 /**
  * Set the global override for whether the client should ever send encrypted
- * messages to unverified devices.  If false, it can still be overridden
- * per-room.  If true, it overrides the per-room settings.
+ * messages to unverified devices.  This provides the default for rooms which
+ * do not specify a value.
  *
- * @param {boolean} value whether to unilaterally blacklist all
- * unverified devices
+ * @param {boolean} value whether to blacklist all unverified devices by default
  */
 Crypto.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
     this._globalBlacklistUnverifiedDevices = value;
 };
 
 /**
- * @return {boolean} whether to unilaterally blacklist all unverified devices
+ * @return {boolean} whether to blacklist all unverified devices by default
  */
 Crypto.prototype.getGlobalBlacklistUnverifiedDevices = function() {
     return this._globalBlacklistUnverifiedDevices;
@@ -245,7 +264,7 @@ Crypto.prototype.uploadDeviceKeys = function() {
 
 /**
  * Stores the current one_time_key count which will be handled later (in a call of
- * _onSyncCompleted). The count is e.g. coming from a /sync response.
+ * onSyncCompleted). The count is e.g. coming from a /sync response.
  *
  * @param {Number} currentCount The current count of one_time_keys to be stored
  */
@@ -307,14 +326,15 @@ function _maybeUploadOneTimeKeys(crypto) {
     function uploadLoop(keyCount) {
         if (keyLimit <= keyCount) {
             // If we don't need to generate any more keys then we are done.
-            return;
+            return Promise.resolve();
         }
 
         const keysThisLoop = Math.min(keyLimit - keyCount, maxKeysPerCycle);
 
         // Ask olm to generate new one time keys, then upload them to synapse.
-        crypto._olmDevice.generateOneTimeKeys(keysThisLoop);
-        return _uploadOneTimeKeys(crypto).then((res) => {
+        return crypto._olmDevice.generateOneTimeKeys(keysThisLoop).then(() => {
+            return _uploadOneTimeKeys(crypto);
+        }).then((res) => {
             if (res.one_time_key_counts && res.one_time_key_counts.signed_curve25519) {
                 // if the response contains a more up to date value use this
                 // for the next loop
@@ -357,7 +377,7 @@ function _maybeUploadOneTimeKeys(crypto) {
 
 // returns a promise which resolves to the response
 async function _uploadOneTimeKeys(crypto) {
-    const oneTimeKeys = crypto._olmDevice.getOneTimeKeys();
+    const oneTimeKeys = await crypto._olmDevice.getOneTimeKeys();
     const oneTimeJson = {};
 
     const promises = [];
@@ -382,7 +402,7 @@ async function _uploadOneTimeKeys(crypto) {
         device_id: crypto._deviceId,
     });
 
-    crypto._olmDevice.markKeysAsPublished();
+    await crypto._olmDevice.markKeysAsPublished();
     return res;
 }
 
@@ -424,6 +444,22 @@ Crypto.prototype.getStoredDevice = function(userId, deviceId) {
     return this._deviceList.getStoredDevice(userId, deviceId);
 };
 
+/**
+ * Save the device list, if necessary
+ *
+ * @param {integer} delay Time in ms before which the save actually happens.
+ *     By default, the save is delayed for a short period in order to batch
+ *     multiple writes, but this behaviour can be disabled by passing 0.
+ *
+ * @return {Promise<bool>} true if the data was saved, false if
+ *     it was not (eg. because no changes were pending). The promise
+ *     will only resolve once the data is saved, so may take some time
+ *     to resolve.
+ */
+Crypto.prototype.saveDeviceList = function(delay) {
+    return this._deviceList.saveIfDirty(delay);
+};
+
 /**
  * Update the blocked/verified state of the given device
  *
@@ -444,7 +480,7 @@ Crypto.prototype.getStoredDevice = function(userId, deviceId) {
 Crypto.prototype.setDeviceVerification = async function(
     userId, deviceId, verified, blocked, known,
 ) {
-    const devices = this._sessionStore.getEndToEndDevicesForUser(userId);
+    const devices = this._deviceList.getRawStoredDevicesForUser(userId);
     if (!devices || !devices[deviceId]) {
         throw new Error("Unknown device " + userId + ":" + deviceId);
     }
@@ -472,7 +508,8 @@ Crypto.prototype.setDeviceVerification = async function(
     if (dev.verified !== verificationStatus || dev.known !== knownStatus) {
         dev.verified = verificationStatus;
         dev.known = knownStatus;
-        this._sessionStore.storeEndToEndDevicesForUser(userId, devices);
+        this._deviceList.storeDevicesForUser(userId, devices);
+        this._deviceList.saveIfDirty();
     }
     return DeviceInfo.fromStorage(dev, deviceId);
 };
@@ -498,7 +535,7 @@ Crypto.prototype.getOlmSessionsForUser = async function(userId) {
     for (let j = 0; j < devices.length; ++j) {
         const device = devices[j];
         const deviceKey = device.getIdentityKey();
-        const sessions = this._olmDevice.getSessionInfoForDevice(deviceKey);
+        const sessions = await this._olmDevice.getSessionInfoForDevice(deviceKey);
 
         result[device.deviceId] = {
             deviceIdKey: deviceKey,
@@ -569,7 +606,6 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
     return device;
 };
 
-
 /**
  * Configure a room to use encryption (ie, save a flag in the sessionstore).
  *
@@ -583,13 +619,11 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
 Crypto.prototype.setRoomEncryption = async function(roomId, config, inhibitDeviceQuery) {
     // if we already have encryption in this room, we should ignore this event
     // (for now at least. maybe we should alert the user somehow?)
-    const existingConfig = this._sessionStore.getEndToEndRoom(roomId);
-    if (existingConfig) {
-        if (JSON.stringify(existingConfig) != JSON.stringify(config)) {
-            console.error("Ignoring m.room.encryption event which requests " +
-                          "a change of config in " + roomId);
-            return;
-        }
+    const existingConfig = this._roomList.getRoomEncryption(roomId);
+    if (existingConfig && JSON.stringify(existingConfig) != JSON.stringify(config)) {
+        console.error("Ignoring m.room.encryption event which requests " +
+                      "a change of config in " + roomId);
+        return;
     }
 
     const AlgClass = algorithms.ENCRYPTION_CLASSES[config.algorithm];
@@ -597,7 +631,7 @@ Crypto.prototype.setRoomEncryption = async function(roomId, config, inhibitDevic
         throw new Error("Unable to encrypt with " + config.algorithm);
     }
 
-    this._sessionStore.storeEndToEndRoom(roomId, config);
+    await this._roomList.setRoomEncryption(roomId, config);
 
     const alg = new AlgClass({
         userId: this._userId,
@@ -675,35 +709,28 @@ Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
     );
 };
 
-/**
- * Whether encryption is enabled for a room.
- * @param {string} roomId the room id to query.
- * @return {bool} whether encryption is enabled.
- */
-Crypto.prototype.isRoomEncrypted = function(roomId) {
-    return Boolean(this._roomEncryptors[roomId]);
-};
-
-
 /**
  * Get a list containing all of the room keys
  *
- * @return {module:client.Promise} a promise which resolves to a list of
- *    session export objects
+ * @return {module:crypto/OlmDevice.MegolmSessionData[]} a list of session export objects
  */
-Crypto.prototype.exportRoomKeys = function() {
-    return Promise.resolve(
-        this._sessionStore.getAllEndToEndInboundGroupSessionKeys().map(
-            (s) => {
-                const sess = this._olmDevice.exportInboundGroupSession(
-                    s.senderKey, s.sessionId,
-                );
+Crypto.prototype.exportRoomKeys = async function() {
+    const exportedSessions = [];
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+            this._cryptoStore.getAllEndToEndInboundGroupSessions(txn, (s) => {
+                if (s === null) return;
 
+                const sess = this._olmDevice.exportInboundGroupSession(
+                    s.senderKey, s.sessionId, s.sessionData,
+                );
                 sess.algorithm = olmlib.MEGOLM_ALGORITHM;
-                return sess;
-            },
-        ),
+                exportedSessions.push(sess);
+            });
+        },
     );
+
+    return exportedSessions;
 };
 
 /**
@@ -717,11 +744,11 @@ Crypto.prototype.importRoomKeys = function(keys) {
         keys, (key) => {
             if (!key.room_id || !key.algorithm) {
                 console.warn("ignoring room key entry with missing fields", key);
-                return;
+                return null;
             }
 
             const alg = this._getRoomDecryptor(key.room_id, key.algorithm);
-            alg.importRoomKey(key);
+            return alg.importRoomKey(key);
         },
     );
 };
@@ -771,25 +798,47 @@ Crypto.prototype.encryptEvent = function(event, room) {
  *
  * @param {MatrixEvent} event
  *
- * @return {Promise} resolves once we have finished decrypting. Rejects with an
- * `algorithms.DecryptionError` if there is a problem decrypting the event.
+ * @return {Promise<module:crypto~EventDecryptionResult>} resolves once we have
+ *  finished decrypting. Rejects with an `algorithms.DecryptionError` if there
+ *  is a problem decrypting the event.
  */
 Crypto.prototype.decryptEvent = function(event) {
+    if (event.isRedacted()) {
+        return Promise.resolve({
+            clearEvent: {
+                room_id: event.getRoomId(),
+                type: "m.room.message",
+                content: {},
+            },
+        });
+    }
     const content = event.getWireContent();
     const alg = this._getRoomDecryptor(event.getRoomId(), content.algorithm);
     return alg.decryptEvent(event);
 };
 
 /**
- * Handle the notification from /sync that a user has updated their device list.
+ * Handle the notification from /sync or /keys/changes that device lists have
+ * been changed.
  *
- * @param {String} userId
+ * @param {Object} syncData Object containing sync tokens associated with this sync
+ * @param {Object} syncDeviceLists device_lists field from /sync, or response from
+ * /keys/changes
  */
-Crypto.prototype.userDeviceListChanged = function(userId) {
-    this._deviceList.invalidateUserDeviceList(userId);
+Crypto.prototype.handleDeviceListChanges = async function(syncData, syncDeviceLists) {
+    // Initial syncs don't have device change lists. We'll either get the complete list
+    // of changes for the interval or will have invalidated everything in willProcessSync
+    if (!syncData.oldSyncToken) return;
 
-    // don't flush the outdated device list yet - we do it once we finish
-    // processing the sync.
+    // Here, we're relying on the fact that we only ever save the sync data after
+    // sucessfully saving the device list data, so we're guaranteed that the device
+    // list store is at least as fresh as the sync token from the sync store, ie.
+    // any device changes received in sync tokens prior to the 'next' token here
+    // have been processed and are reflected in the current device list.
+    // If we didn't make this assumption, we'd have to use the /keys/changes API
+    // to get key changes between the sync token in the device list and the 'old'
+    // sync token used here to make sure we didn't miss any.
+    this._evalDeviceListChanges(syncDeviceLists);
 };
 
 /**
@@ -814,9 +863,11 @@ Crypto.prototype.requestRoomKey = function(requestBody, recipients) {
  *
  * @param {module:crypto~RoomKeyRequestBody} requestBody
  *    parameters to match for cancellation
+ * @param {boolean} andResend
+ *    if true, resend the key request after cancelling.
  */
-Crypto.prototype.cancelRoomKeyRequest = function(requestBody) {
-    this._outgoingRoomKeyRequestManager.cancelRoomKeyRequest(requestBody)
+Crypto.prototype.cancelRoomKeyRequest = function(requestBody, andResend) {
+    this._outgoingRoomKeyRequestManager.cancelRoomKeyRequest(requestBody, andResend)
     .catch((e) => {
         console.warn("Error clearing pending room key requests", e);
     }).done();
@@ -833,7 +884,7 @@ Crypto.prototype.onCryptoEvent = async function(event) {
 
     try {
         // inhibit the device list refresh for now - it will happen once we've
-        // finished processing the sync, in _onSyncCompleted.
+        // finished processing the sync, in onSyncCompleted.
         await this.setRoomEncryption(roomId, content, true);
     } catch (e) {
         console.error("Error configuring encryption in room " + roomId +
@@ -841,6 +892,22 @@ Crypto.prototype.onCryptoEvent = async function(event) {
     }
 };
 
+/**
+ * Called before the result of a sync is procesed
+ *
+ * @param {Object} syncData  the data from the 'MatrixClient.sync' event
+ */
+Crypto.prototype.onSyncWillProcess = async function(syncData) {
+    if (!syncData.oldSyncToken) {
+        // If there is no old sync token, we start all our tracking from
+        // scratch, so mark everything as untracked. onCryptoEvent will
+        // be called for all e2e rooms during the processing of the sync,
+        // at which point we'll start tracking all the users of that room.
+        console.log("Initial sync performed - resetting device tracking state");
+        this._deviceList.stopTrackingAllDeviceLists();
+    }
+};
+
 /**
  * handle the completion of a /sync
  *
@@ -849,45 +916,11 @@ Crypto.prototype.onCryptoEvent = async function(event) {
  *
  * @param {Object} syncData  the data from the 'MatrixClient.sync' event
  */
-Crypto.prototype._onSyncCompleted = function(syncData) {
+Crypto.prototype.onSyncCompleted = async function(syncData) {
     const nextSyncToken = syncData.nextSyncToken;
 
-    if (!syncData.oldSyncToken) {
-        console.log("Completed initial sync");
-
-        // if we have a deviceSyncToken, we can tell the deviceList to
-        // invalidate devices which have changed since then.
-        const oldSyncToken = this._sessionStore.getEndToEndDeviceSyncToken();
-        if (oldSyncToken !== null) {
-            this._initialDeviceListInvalidationPending = true;
-            this._invalidateDeviceListsSince(
-                oldSyncToken, nextSyncToken,
-            ).catch((e) => {
-                // if that failed, we fall back to invalidating everyone.
-                console.warn("Error fetching changed device list", e);
-                this._deviceList.invalidateAllDeviceLists();
-            }).done(() => {
-                this._initialDeviceListInvalidationPending = false;
-                this._deviceList.lastKnownSyncToken = nextSyncToken;
-                this._deviceList.refreshOutdatedDeviceLists();
-            });
-        } else {
-            // otherwise, we have to invalidate all devices for all users we
-            // are tracking.
-            console.log("Completed first initialsync; invalidating all " +
-                        "device list caches");
-            this._deviceList.invalidateAllDeviceLists();
-        }
-    }
-
-    if (!this._initialDeviceListInvalidationPending) {
-        // we can now store our sync token so that we can get an update on
-        // restart rather than having to invalidate everyone.
-        //
-        // (we don't really need to do this on every sync - we could just
-        // do it periodically)
-        this._sessionStore.storeEndToEndDeviceSyncToken(nextSyncToken);
-    }
+    this._deviceList.setSyncToken(syncData.nextSyncToken);
+    this._deviceList.saveIfDirty();
 
     // catch up on any new devices we got told about during the sync.
     this._deviceList.lastKnownSyncToken = nextSyncToken;
@@ -904,31 +937,47 @@ Crypto.prototype._onSyncCompleted = function(syncData) {
 };
 
 /**
- * Ask the server which users have new devices since a given token,
- * and invalidate them
+ * Trigger the appropriate invalidations and removes for a given
+ * device list
  *
- * @param {String} oldSyncToken
- * @param {String} lastKnownSyncToken
- *
- * @returns {Promise} resolves once the query is complete. Rejects if the
- *   keyChange query fails.
+ * @param {Object} deviceLists device_lists field from /sync, or response from
+ * /keys/changes
  */
-Crypto.prototype._invalidateDeviceListsSince = function(
-    oldSyncToken, lastKnownSyncToken,
-) {
-    return this._baseApis.getKeyChanges(
-        oldSyncToken, lastKnownSyncToken,
-    ).then((r) => {
-        console.log("got key changes since", oldSyncToken, ":", r.changed);
-
-        if (!r.changed || !Array.isArray(r.changed)) {
-            return;
-        }
-
-        r.changed.forEach((u) => {
+Crypto.prototype._evalDeviceListChanges = async function(deviceLists) {
+    if (deviceLists.changed && Array.isArray(deviceLists.changed)) {
+        deviceLists.changed.forEach((u) => {
             this._deviceList.invalidateUserDeviceList(u);
         });
-    });
+    }
+
+    if (deviceLists.left && Array.isArray(deviceLists.left)) {
+        // Check we really don't share any rooms with these users
+        // any more: the server isn't required to give us the
+        // exact correct set.
+        const e2eUserIds = new Set(this._getE2eUsers());
+
+        deviceLists.left.forEach((u) => {
+            if (!e2eUserIds.has(u)) {
+                this._deviceList.stopTrackingDeviceList(u);
+            }
+        });
+    }
+};
+
+/**
+ * Get a list of all the IDs of users we share an e2e room with
+ *
+ * @returns {string[]} List of user IDs
+ */
+Crypto.prototype._getE2eUsers = function() {
+    const e2eUserIds = [];
+    for (const room of this._getE2eRooms()) {
+        const members = room.getJoinedMembers();
+        for (const member of members) {
+            e2eUserIds.push(member.userId);
+        }
+    }
+    return e2eUserIds;
 };
 
 /**
@@ -1239,7 +1288,7 @@ Crypto.prototype._signObject = async function(obj) {
     const sigs = {};
     sigs[this._userId] = {};
     sigs[this._userId]["ed25519:" + this._deviceId] =
-        this._olmDevice.sign(anotherjson.stringify(obj));
+        await this._olmDevice.sign(anotherjson.stringify(obj));
     obj.signatures = sigs;
 };
 
@@ -1294,6 +1343,27 @@ class IncomingRoomKeyRequestCancellation {
     }
 }
 
+/**
+ * The result of a (successful) call to decryptEvent.
+ *
+ * @typedef {Object} EventDecryptionResult
+ *
+ * @property {Object} clearEvent The plaintext payload for the event
+ *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
+ *
+ * @property {?string} senderCurve25519Key Key owned by the sender of this
+ *    event.  See {@link module:models/event.MatrixEvent#getSenderKey}.
+ *
+ * @property {?string} claimedEd25519Key ed25519 key claimed by the sender of
+ *    this event. See
+ *    {@link module:models/event.MatrixEvent#getClaimedEd25519Key}.
+ *
+ * @property {?Array<string>} forwardingCurve25519KeyChain list of curve25519
+ *     keys involved in telling us about the senderCurve25519Key and
+ *     claimedEd25519Key. See
+ *     {@link module:models/event.MatrixEvent#getForwardingCurve25519KeyChain}.
+ */
+
 /**
  * Fires when we receive a room key request
  *
@@ -1308,5 +1378,24 @@ class IncomingRoomKeyRequestCancellation {
  * @param {module:crypto~IncomingRoomKeyRequestCancellation} req
  */
 
+/**
+ * Fires when the app may wish to warn the user about something related
+ * the end-to-end crypto.
+ *
+ * Comes with a type which is one of:
+ * * CRYPTO_WARNING_ACCOUNT_MIGRATED: Account data has been migrated from an older
+ *       version of the store in such a way that older clients will no longer be
+ *       able to read it. The app may wish to warn the user against going back to
+ *       an older version of the app.
+ * * CRYPTO_WARNING_OLD_VERSION_DETECTED: js-sdk has detected that an older version
+ *       of js-sdk has been run against the same store after a migration has been
+ *       performed. This is likely have caused unexpected behaviour in the old
+ *       version. For example, the old version and the new version may have two
+ *       different identity keys.
+ *
+ * @event module:client~MatrixClient#"crypto.warning"
+ * @param {string} type One of the strings listed above
+ */
+
 /** */
 module.exports = Crypto;

src/crypto/olmlib.js

@@ -58,7 +58,7 @@ module.exports.encryptMessageForDevice = async function(
     payloadFields,
 ) {
     const deviceKey = recipientDevice.getIdentityKey();
-    const sessionId = olmDevice.getSessionIdForDevice(deviceKey);
+    const sessionId = await olmDevice.getSessionIdForDevice(deviceKey);
     if (sessionId === null) {
         // If we don't have a session for a device then
         // we can't encrypt a message for it.
@@ -102,7 +102,7 @@ module.exports.encryptMessageForDevice = async function(
 
     utils.extend(payload, payloadFields);
 
-    resultsObject[deviceKey] = olmDevice.encryptMessage(
+    resultsObject[deviceKey] = await olmDevice.encryptMessage(
         deviceKey, sessionId, JSON.stringify(payload),
     );
 };
@@ -139,7 +139,7 @@ module.exports.ensureOlmSessionsForDevices = async function(
             const deviceInfo = devices[j];
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
-            const sessionId = olmDevice.getSessionIdForDevice(key);
+            const sessionId = await olmDevice.getSessionIdForDevice(key);
             if (sessionId === null) {
                 devicesWithoutSession.push([userId, deviceId]);
             }
@@ -228,7 +228,7 @@ async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceIn
 
     let sid;
     try {
-        sid = olmDevice.createOutboundSession(
+        sid = await olmDevice.createOutboundSession(
             deviceInfo.getIdentityKey(), oneTimeKey.key,
         );
     } catch (e) {

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -1,7 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import Promise from 'bluebird';
 import utils from '../../utils';
 
-export const VERSION = 1;
+export const VERSION = 6;
 
 /**
  * Implementation of a CryptoStore which is backed by an existing
@@ -62,9 +79,9 @@ export class Backend {
                 `enqueueing key request for ${requestBody.room_id} / ` +
                     requestBody.session_id,
             );
+            txn.oncomplete = () => { deferred.resolve(request); };
             const store = txn.objectStore("outgoingRoomKeyRequests");
             store.add(request);
-            txn.onsuccess = () => { deferred.resolve(request); };
         });
 
         return deferred.promise;
@@ -257,6 +274,200 @@ export class Backend {
         };
         return promiseifyTxn(txn);
     }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeAccount(txn, newData) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(newData, "-");
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const countReq = objectStore.count();
+        countReq.onsuccess = function() {
+            func(countReq.result);
+        };
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const idx = objectStore.index("deviceKey");
+        const getReq = idx.openCursor(deviceKey);
+        const results = {};
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                results[cursor.value.sessionId] = cursor.value.session;
+                cursor.continue();
+            } else {
+                try {
+                    func(results);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.get([deviceKey, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const objectStore = txn.objectStore("sessions");
+        objectStore.put({deviceKey, sessionId, session});
+    }
+
+    // Inbound group sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.get([senderCurve25519Key, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                try {
+                    func({
+                        senderKey: cursor.value.senderCurve25519Key,
+                        sessionId: cursor.value.sessionId,
+                        sessionData: cursor.value.session,
+                    });
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const addReq = objectStore.add({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+        addReq.onerror = (ev) => {
+            if (addReq.error.name === 'ConstraintError') {
+                // This stops the error from triggering the txn's onerror
+                ev.stopPropagation();
+                // ...and this stops it from aborting the transaction
+                ev.preventDefault();
+                console.log(
+                    "Ignoring duplicate inbound group session: " +
+                    senderCurve25519Key + " / " + sessionId,
+                );
+            } else {
+                abortWithException(txn, new Error(
+                    "Failed to add inbound group session: " + addReq.error,
+                ));
+            }
+        };
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        objectStore.put({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        const objectStore = txn.objectStore("device_data");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        const objectStore = txn.objectStore("device_data");
+        objectStore.put(deviceData, "-");
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        const objectStore = txn.objectStore("rooms");
+        objectStore.put(roomInfo, roomId);
+    }
+
+    getEndToEndRooms(txn, func) {
+        const rooms = {};
+        const objectStore = txn.objectStore("rooms");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                rooms[cursor.key] = cursor.value;
+                cursor.continue();
+            } else {
+                try {
+                    func(rooms);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    doTxn(mode, stores, func) {
+        const txn = this._db.transaction(stores, mode);
+        const promise = promiseifyTxn(txn);
+        const result = func(txn);
+        return promise.then(() => {
+            return result;
+        });
+    }
 }
 
 export function upgradeDatabase(db, oldVersion) {
@@ -267,6 +478,26 @@ export function upgradeDatabase(db, oldVersion) {
     if (oldVersion < 1) { // The database did not previously exist.
         createDatabase(db);
     }
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", {
+            keyPath: ["deviceKey", "sessionId"],
+        });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
     // Expand as needed.
 }
 
@@ -283,9 +514,37 @@ function createDatabase(db) {
     outgoingRoomKeyRequestsStore.createIndex("state", "state");
 }
 
+/*
+ * Aborts a transaction with a given exception
+ * The transaction promise will be rejected with this exception.
+ */
+function abortWithException(txn, e) {
+    // We cheekily stick our exception onto the transaction object here
+    // We could alternatively make the thing we pass back to the app
+    // an object containing the transaction and exception.
+    txn._mx_abortexception = e;
+    try {
+        txn.abort();
+    } catch (e) {
+        // sometimes we won't be able to abort the transaction
+        // (ie. if it's aborted or completed)
+    }
+}
+
 function promiseifyTxn(txn) {
     return new Promise((resolve, reject) => {
-        txn.oncomplete = resolve;
-        txn.onerror = reject;
+        txn.oncomplete = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            resolve();
+        };
+        txn.onerror = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            reject();
+        };
+        txn.onabort = () => reject(txn._mx_abortexception);
     });
 }

src/crypto/store/indexeddb-crypto-store.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +17,7 @@ limitations under the License.
 
 import Promise from 'bluebird';
 
+import LocalStorageCryptoStore from './localStorage-crypto-store';
 import MemoryCryptoStore from './memory-crypto-store';
 import * as IndexedDBCryptoStoreBackend from './indexeddb-crypto-store-backend';
 
@@ -46,7 +48,7 @@ export default class IndexedDBCryptoStore {
 
     /**
      * Ensure the database exists and is up-to-date, or fall back to
-     * an in-memory store.
+     * a local storage or in-memory store.
      *
      * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
      * or a MemoryCryptoStore
@@ -93,7 +95,12 @@ export default class IndexedDBCryptoStore {
         }).catch((e) => {
             console.warn(
                 `unable to connect to indexeddb ${this._dbName}` +
-                    `: falling back to in-memory store: ${e}`,
+                    `: falling back to localStorage store: ${e}`,
+            );
+            return new LocalStorageCryptoStore(global.localStorage);
+        }).catch((e) => {
+            console.warn(
+                `unable to open localStorage: falling back to in-memory store: ${e}`,
             );
             return new MemoryCryptoStore();
         });
@@ -220,4 +227,216 @@ export default class IndexedDBCryptoStore {
             return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
         });
     }
+
+    // Olm Account
+
+    /*
+     * Get the account pickle from the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the account pickle
+     */
+    getAccount(txn, func) {
+        this._backendPromise.value().getAccount(txn, func);
+    }
+
+    /*
+     * Write the account pickle to the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} newData The new account pickle to store.
+     */
+    storeAccount(txn, newData) {
+        this._backendPromise.value().storeAccount(txn, newData);
+    }
+
+    // Olm sessions
+
+    /**
+     * Returns the number of end-to-end sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(int)} func Called with the count of sessions
+     */
+    countEndToEndSessions(txn, func) {
+        this._backendPromise.value().countEndToEndSessions(txn, func);
+    }
+
+    /**
+     * Retrieve a specific end-to-end session between the logged-in user
+     * and another device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID of the session to retrieve
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndSession(deviceKey, sessionId, txn, func);
+    }
+
+    /**
+     * Retrieve the end-to-end sessions between the logged-in user and another
+     * device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSessions(deviceKey, txn, func) {
+        this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func);
+    }
+
+    /**
+     * Store a session between the logged-in user and another device
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID for this end-to-end session.
+     * @param {string} session Base64 encoded end-to-end session.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        this._backendPromise.value().storeEndToEndSession(
+            deviceKey, sessionId, session, txn,
+        );
+    }
+
+    // Inbound group saessions
+
+    /**
+     * Retrieve the end-to-end inbound group session for a given
+     * server key and session ID
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
+    }
+
+    /**
+     * Fetches all inbound group sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called once for each group session
+     *     in the store with an object having keys {senderKey, sessionId,
+     *     sessionData}, then once with null to indicate the end of the list.
+     */
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndInboundGroupSessions(txn, func);
+    }
+
+    /**
+     * Adds an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, the session will not be added.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    /**
+     * Writes an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, it will be overwritten.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    // End-to-end device tracking
+
+    /**
+     * Store the state of all tracked devices
+     * This contains devices for each user, a tracking state for each user
+     * and a sync token matching the point in time the snapshot represents.
+     * These all need to be written out in full each time such that the snapshot
+     * is always consistent, so they are stored in one object.
+     *
+     * @param {Object} deviceData
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._backendPromise.value().storeEndToEndDeviceData(deviceData, txn);
+    }
+
+    /**
+     * Get the state of all tracked devices
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the
+     *     device data
+     */
+    getEndToEndDeviceData(txn, func) {
+        this._backendPromise.value().getEndToEndDeviceData(txn, func);
+    }
+
+    // End to End Rooms
+
+    /**
+     * Store the end-to-end state for a room.
+     * @param {string} roomId The room's ID.
+     * @param {object} roomInfo The end-to-end info for the room.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._backendPromise.value().storeEndToEndRoom(roomId, roomInfo, txn);
+    }
+
+    /**
+     * Get an object of roomId->roomInfo for all e2e rooms in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the end to end encrypted rooms
+     */
+    getEndToEndRooms(txn, func) {
+        this._backendPromise.value().getEndToEndRooms(txn, func);
+    }
+
+    /**
+     * Perform a transaction on the crypto store. Any store methods
+     * that require a transaction (txn) object to be passed in may
+     * only be called within a callback of either this function or
+     * one of the store functions operating on the same transaction.
+     *
+     * @param {string} mode 'readwrite' if you need to call setter
+     *     functions with this transaction. Otherwise, 'readonly'.
+     * @param {string[]} stores List IndexedDBCryptoStore.STORE_*
+     *     options representing all types of object that will be
+     *     accessed or written to with this transaction.
+     * @param {function(*)} func Function called with the
+     *     transaction object: an opaque object that should be passed
+     *     to store functions.
+     * @return {Promise} Promise that resolves with the result of the `func`
+     *     when the transaction is complete. If the backend is
+     *     async (ie. the indexeddb backend) any of the callback
+     *     functions throwing an exception will cause this promise to
+     *     reject with that exception. On synchronous backends, the
+     *     exception will propagate to the caller of the getFoo method.
+     */
+    doTxn(mode, stores, func) {
+        return this._connect().then((backend) => {
+            return backend.doTxn(mode, stores, func);
+        });
+    }
 }
+
+IndexedDBCryptoStore.STORE_ACCOUNT = 'account';
+IndexedDBCryptoStore.STORE_SESSIONS = 'sessions';
+IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS = 'inbound_group_sessions';
+IndexedDBCryptoStore.STORE_DEVICE_DATA = 'device_data';
+IndexedDBCryptoStore.STORE_ROOMS = 'rooms';

src/crypto/store/localStorage-crypto-store.js

@@ -0,0 +1,210 @@
+/*
+Copyright 2017, 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+import MemoryCryptoStore from './memory-crypto-store.js';
+
+/**
+ * Internal module. Partial localStorage backed storage for e2e.
+ * This is not a full crypto store, just the in-memory store with
+ * some things backed by localStorage. It exists because indexedDB
+ * is broken in Firefox private mode or set to, "will not remember
+ * history".
+ *
+ * @module
+ */
+
+const E2E_PREFIX = "crypto.";
+const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
+const KEY_DEVICE_DATA = E2E_PREFIX + "device_data";
+const KEY_INBOUND_SESSION_PREFIX = E2E_PREFIX + "inboundgroupsessions/";
+const KEY_ROOMS_PREFIX = E2E_PREFIX + "rooms/";
+
+function keyEndToEndSessions(deviceKey) {
+    return E2E_PREFIX + "sessions/" + deviceKey;
+}
+
+function keyEndToEndInboundGroupSession(senderKey, sessionId) {
+    return KEY_INBOUND_SESSION_PREFIX + senderKey + "/" + sessionId;
+}
+
+function keyEndToEndRoomsPrefix(roomId) {
+    return KEY_ROOMS_PREFIX + roomId;
+}
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class LocalStorageCryptoStore extends MemoryCryptoStore {
+    constructor(webStore) {
+        super();
+        this.store = webStore;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        let count = 0;
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) ++count;
+        }
+        func(count);
+    }
+
+    _getEndToEndSessions(deviceKey, txn, func) {
+        return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const sessions = this._getEndToEndSessions(deviceKey);
+        func(sessions[sessionId] || {});
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._getEndToEndSessions(deviceKey) || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const sessions = this._getEndToEndSessions(deviceKey) || {};
+        sessions[sessionId] = session;
+        setJsonItem(
+            this.store, keyEndToEndSessions(deviceKey), sessions,
+        );
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        ));
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                // we can't use split, as the components we are trying to split out
+                // might themselves contain '/' characters. We rely on the
+                // senderKey being a (32-byte) curve25519 key, base64-encoded
+                // (hence 43 characters long).
+
+                func({
+                    senderKey: key.substr(KEY_INBOUND_SESSION_PREFIX.length, 43),
+                    sessionId: key.substr(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    sessionData: getJsonItem(this.store, key),
+                });
+            }
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const existing = getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        );
+        if (!existing) {
+            this.storeEndToEndInboundGroupSession(
+                senderCurve25519Key, sessionId, sessionData, txn,
+            );
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        setJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+            sessionData,
+        );
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        func(getJsonItem(
+            this.store, KEY_DEVICE_DATA,
+        ));
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        setJsonItem(
+            this.store, KEY_DEVICE_DATA, deviceData,
+        );
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        setJsonItem(
+            this.store, keyEndToEndRoomsPrefix(roomId), roomInfo,
+        );
+    }
+
+    getEndToEndRooms(txn, func) {
+        const result = {};
+        const prefix = keyEndToEndRoomsPrefix('');
+
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(prefix)) {
+                const roomId = key.substr(prefix.length);
+                result[roomId] = getJsonItem(this.store, key);
+            }
+        }
+        func(result);
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
+        return Promise.resolve();
+    }
+
+    // Olm account
+
+    getAccount(txn, func) {
+        const account = getJsonItem(this.store, KEY_END_TO_END_ACCOUNT);
+        func(account);
+    }
+
+    storeAccount(txn, newData) {
+        setJsonItem(
+            this.store, KEY_END_TO_END_ACCOUNT, newData,
+        );
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
+}
+
+function getJsonItem(store, key) {
+    try {
+        // if the key is absent, store.getItem() returns null, and
+        // JSON.parse(null) === null, so this returns null.
+        return JSON.parse(store.getItem(key));
+    } catch (e) {
+        console.log("Error: Failed to get key %s: %s", key, e.stack || e);
+        console.log(e.stack);
+    }
+    return null;
+}
+
+function setJsonItem(store, key, val) {
+    store.setItem(key, JSON.stringify(val));
+}

src/crypto/store/memory-crypto-store.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -30,6 +31,16 @@ import utils from '../../utils';
 export default class MemoryCryptoStore {
     constructor() {
         this._outgoingRoomKeyRequests = [];
+        this._account = null;
+
+        // Map of {devicekey -> {sessionId -> session pickle}}
+        this._sessions = {};
+        // Map of {senderCurve25519Key+'/'+sessionId -> session data object}
+        this._inboundGroupSessions = {};
+        // Opaque device data object
+        this._deviceData = null;
+        // roomId -> Opaque roomInfo object
+        this._rooms = {};
     }
 
     /**
@@ -196,4 +207,95 @@ export default class MemoryCryptoStore {
 
         return Promise.resolve(null);
     }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        func(this._account);
+    }
+
+    storeAccount(txn, newData) {
+        this._account = newData;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        return Object.keys(this._sessions).length;
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const deviceSessions = this._sessions[deviceKey] || {};
+        func(deviceSessions[sessionId] || null);
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._sessions[deviceKey] || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        let deviceSessions = this._sessions[deviceKey];
+        if (deviceSessions === undefined) {
+            deviceSessions = {};
+            this._sessions[deviceKey] = deviceSessions;
+        }
+        deviceSessions[sessionId] = session;
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] || null);
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (const key of Object.keys(this._inboundGroupSessions)) {
+            // we can't use split, as the components we are trying to split out
+            // might themselves contain '/' characters. We rely on the
+            // senderKey being a (32-byte) curve25519 key, base64-encoded
+            // (hence 43 characters long).
+
+            func({
+                senderKey: key.substr(0, 43),
+                sessionId: key.substr(44),
+                sessionData: this._inboundGroupSessions[key],
+            });
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const k = senderCurve25519Key+'/'+sessionId;
+        if (this._inboundGroupSessions[k] === undefined) {
+            this._inboundGroupSessions[k] = sessionData;
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] = sessionData;
+    }
+
+    // Device Data
+
+    getEndToEndDeviceData(txn, func) {
+        func(this._deviceData);
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._deviceData = deviceData;
+    }
+
+    // E2E rooms
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._rooms[roomId] = roomInfo;
+    }
+
+    getEndToEndRooms(txn, func) {
+        func(this._rooms);
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
 }

src/filter-component.js

@@ -97,11 +97,12 @@ FilterComponent.prototype._checkFields =
     };
 
     const self = this;
-    Object.keys(literal_keys).forEach(function(name) {
+    for (let n=0; n < Object.keys(literal_keys).length; n++) {
+        const name = Object.keys(literal_keys)[n];
         const match_func = literal_keys[name];
         const not_name = "not_" + name;
         const disallowed_values = self[not_name];
-        if (disallowed_values.map(match_func)) {
+        if (disallowed_values.filter(match_func).length > 0) {
             return false;
         }
 
@@ -111,7 +112,7 @@ FilterComponent.prototype._checkFields =
                 return false;
             }
         }
-    });
+    }
 
     const contains_url_filter = this.filter_json.contains_url;
     if (contains_url_filter !== undefined) {

src/interactive-auth.js

@@ -293,7 +293,9 @@ InteractiveAuth.prototype = {
             },
         );
         if (!background) {
-            prom = prom.catch(this._completionDeferred.reject);
+            prom = prom.catch((e) => {
+                this._completionDeferred.reject(e);
+            });
         } else {
             // We ignore all failures here (even non-UI auth related ones)
             // since we don't want to suddenly fail if the internet connection

src/models/event-timeline.js

@@ -20,8 +20,6 @@ limitations under the License.
  */
 
 const RoomState = require("./room-state");
-const utils = require("../utils");
-const MatrixEvent = require("./event").MatrixEvent;
 
 /**
  * Construct a new EventTimeline
@@ -88,19 +86,23 @@ EventTimeline.prototype.initialiseState = function(stateEvents) {
         throw new Error("Cannot initialise state after events are added");
     }
 
-    // we deep-copy the events here, in case they get changed later - we don't
-    // want changes to the start state leaking through to the end state.
-    const oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEvents.map(function(mxEvent) {
-                return mxEvent.event;
-            }),
-        ),
-    function(ev) {
-        return new MatrixEvent(ev);
-    });
+    // We previously deep copied events here and used different copies in
+    // the oldState and state events: this decision seems to date back
+    // quite a way and was apparently made to fix a bug where modifications
+    // made to the start state leaked through to the end state.
+    // This really shouldn't be possible though: the events themselves should
+    // not change. Duplicating the events uses a lot of extra memory,
+    // so we now no longer do it. To assert that they really do never change,
+    // freeze them! Note that we can't do this for events in general:
+    // although it looks like the only things preventing us are the
+    // 'status' flag, forwardLooking (which is only set once when adding to the
+    // timeline) and possibly the sender (which seems like it should never be
+    // reset but in practice causes a lot of the tests to break).
+    for (const e of stateEvents) {
+        Object.freeze(e);
+    }
 
-    this._startState.setStateEvents(oldStateEvents);
+    this._startState.setStateEvents(stateEvents);
     this._endState.setStateEvents(stateEvents);
 };
 

src/models/event.js

@@ -21,11 +21,9 @@ limitations under the License.
  * @module models/event
  */
 
-const EventEmitter = require("events").EventEmitter;
-
-const utils = require('../utils.js');
-
 import Promise from 'bluebird';
+import {EventEmitter} from 'events';
+import utils from '../utils.js';
 
 /**
  * Enum for event statuses.
@@ -82,7 +80,10 @@ module.exports.MatrixEvent = function MatrixEvent(
     // intern the values of matrix events to force share strings and reduce the
     // amount of needless string duplication. This can save moderate amounts of
     // memory (~10% on a 350MB heap).
-    ["state_key", "type", "sender", "room_id"].forEach((prop) => {
+    // 'membership' at the event level (rather than the content level) is a legacy
+    // field that Riot never otherwise looks at, but it will still take up a lot
+    // of space if we don't intern it.
+    ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
         if (!event[prop]) {
             return;
         }
@@ -110,8 +111,6 @@ module.exports.MatrixEvent = function MatrixEvent(
     this.error = null;
     this.forwardLooking = true;
     this._pushActions = null;
-    this._date = this.event.origin_server_ts ?
-        new Date(this.event.origin_server_ts) : null;
 
     this._clearEvent = {};
 
@@ -131,8 +130,10 @@ module.exports.MatrixEvent = function MatrixEvent(
      */
     this._forwardingCurve25519KeyChain = [];
 
-    /* flag to indicate if we have a process decrypting this event */
-    this._decrypting = false;
+    /* if we have a process decrypting this event, a Promise which resolves
+     * when it is finished. Normally null.
+     */
+    this._decryptionPromise = null;
 
     /* flag to indicate if we should retry decrypting this event after the
      * first attempt (eg, we have received new data which means that a second
@@ -204,7 +205,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Date} The event date, e.g. <code>new Date(1433502692297)</code>
      */
     getDate: function() {
-        return this._date;
+        return this.event.origin_server_ts ? new Date(this.event.origin_server_ts) : null;
     },
 
     /**
@@ -315,7 +316,20 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {boolean} True if this event is currently being decrypted, else false.
      */
     isBeingDecrypted: function() {
-        return this._decrypting;
+        return this._decryptionPromise != null;
+    },
+
+    /**
+     * Check if this event is an encrypted event which we failed to decrypt
+     *
+     * (This implies that we might retry decryption at some point in the future)
+     *
+     * @return {boolean} True if this event is an encrypted event which we
+     *     couldn't decrypt.
+     */
+    isDecryptionFailure: function() {
+        return this._clearEvent && this._clearEvent.content &&
+            this._clearEvent.content.msgtype === "m.bad.encrypted";
     },
 
     /**
@@ -326,13 +340,12 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @internal
      *
      * @param {module:crypto} crypto crypto module
+     *
+     * @returns {Promise} promise which resolves (to undefined) when the decryption
+     * attempt is completed.
      */
-    attemptDecryption: function(crypto) {
-        if (!crypto) {
-            this._badEncryptedMessage("Encryption not enabled");
-            return;
-        }
-
+    attemptDecryption: async function(crypto) {
+        // start with a couple of sanity checks.
         if (!this.isEncrypted()) {
             throw new Error("Attempt to decrypt event which isn't encrypted");
         }
@@ -347,60 +360,127 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             );
         }
 
-        if (this._decrypting) {
+        // if we already have a decryption attempt in progress, then it may
+        // fail because it was using outdated info. We now have reason to
+        // succeed where it failed before, but we don't want to have multiple
+        // attempts going at the same time, so just set a flag that says we have
+        // new info.
+        //
+        if (this._decryptionPromise) {
             console.log(
                 `Event ${this.getId()} already being decrypted; queueing a retry`,
             );
             this._retryDecryption = true;
-            return;
+            return this._decryptionPromise;
         }
 
-        this._decrypting = true;
-
-        this._doDecryption(crypto).finally(() => {
-            this._decrypting = false;
-            this._retryDecryption = false;
-        });
+        this._decryptionPromise = this._decryptionLoop(crypto);
+        return this._decryptionPromise;
     },
 
-    _doDecryption: function(crypto) {
-        return Promise.try(() => {
-            return crypto.decryptEvent(this);
-        }).catch((e) => {
-            if (e.name !== "DecryptionError") {
-                // not a decryption error: log the whole exception as an error.
-                console.error(
-                    `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
-                );
-                return null;
-            } else if (this._retryDecryption) {
-                // decryption error, but we have a retry queued.
-                console.log(
-                    `Got error decrypting event (id=${this.getId()}), but retrying`,
-                );
-                this._retryDecryption = false;
-                return this._doDecryption(crypto);
-            } else {
+    /**
+     * Cancel any room key request for this event and resend another.
+     *
+     * @param {module:crypto} crypto crypto module
+     */
+    cancelAndResendKeyRequest: function(crypto) {
+        const wireContent = this.getWireContent();
+        crypto.cancelRoomKeyRequest({
+            algorithm: wireContent.algorithm,
+            room_id: this.getRoomId(),
+            session_id: wireContent.session_id,
+            sender_key: wireContent.sender_key,
+        }, true);
+    },
+
+    _decryptionLoop: async function(crypto) {
+        // make sure that this method never runs completely synchronously.
+        // (doing so would mean that we would clear _decryptionPromise *before*
+        // it is set in attemptDecryption - and hence end up with a stuck
+        // `_decryptionPromise`).
+        await Promise.resolve();
+
+        while (true) {
+            this._retryDecryption = false;
+
+            let res;
+            try {
+                if (!crypto) {
+                    res = this._badEncryptedMessage("Encryption not enabled");
+                } else {
+                    res = await crypto.decryptEvent(this);
+                }
+            } catch (e) {
+                if (e.name !== "DecryptionError") {
+                    // not a decryption error: log the whole exception as an error
+                    // (and don't bother with a retry)
+                    console.error(
+                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                    );
+                    this._decryptionPromise = null;
+                    this._retryDecryption = false;
+                    return;
+                }
+
+                // see if we have a retry queued.
+                //
+                // NB: make sure to keep this check in the same tick of the
+                //   event loop as `_decryptionPromise = null` below - otherwise we
+                //   risk a race:
+                //
+                //   * A: we check _retryDecryption here and see that it is
+                //        false
+                //   * B: we get a second call to attemptDecryption, which sees
+                //        that _decryptionPromise is set so sets
+                //        _retryDecryption
+                //   * A: we continue below, clear _decryptionPromise, and
+                //        never do the retry.
+                //
+                if (this._retryDecryption) {
+                    // decryption error, but we have a retry queued.
+                    console.log(
+                        `Got error decrypting event (id=${this.getId()}: ` +
+                        `${e}), but retrying`,
+                    );
+                    continue;
+                }
+
                 // decryption error, no retries queued. Warn about the error and
                 // set it to m.bad.encrypted.
                 console.warn(
-                    `Error decrypting event (id=${this.getId()}): ${e}`,
+                    `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
                 );
 
-                this._badEncryptedMessage(e.message);
-                return null;
+                res = this._badEncryptedMessage(e.message);
             }
-        });
+
+            // at this point, we've either successfully decrypted the event, or have given up
+            // (and set res to a 'badEncryptedMessage'). Either way, we can now set the
+            // cleartext of the event and raise Event.decrypted.
+            //
+            // make sure we clear '_decryptionPromise' before sending the 'Event.decrypted' event,
+            // otherwise the app will be confused to see `isBeingDecrypted` still set when
+            // there isn't an `Event.decrypted` on the way.
+            //
+            // see also notes on _retryDecryption above.
+            //
+            this._decryptionPromise = null;
+            this._retryDecryption = false;
+            this._setClearData(res);
+            return;
+        }
     },
 
     _badEncryptedMessage: function(reason) {
-        this.setClearData({
-            type: "m.room.message",
-            content: {
-                msgtype: "m.bad.encrypted",
-                body: "** Unable to decrypt: " + reason + " **",
+        return {
+            clearEvent: {
+                type: "m.room.message",
+                content: {
+                    msgtype: "m.bad.encrypted",
+                    body: "** Unable to decrypt: " + reason + " **",
+                },
             },
-        });
+        };
     },
 
     /**
@@ -412,29 +492,17 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      *
      * @fires module:models/event.MatrixEvent#"Event.decrypted"
      *
-     * @param {Object} clearEvent The plaintext payload for the event
-     *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
-     *
-     * @param {string=} senderCurve25519Key Key owned by the sender of this event.
-     *    See {@link module:models/event.MatrixEvent#getSenderKey}.
-     *
-     * @param {string=} claimedEd25519Key ed25519 key claimed by the sender of
-     *    this event. See {@link module:models/event.MatrixEvent#getClaimedEd25519Key}.
-     *
-     * @param {Array<string>=} forwardingCurve25519KeyChain list of curve25519 keys
-     *     involved in telling us about the senderCurve25519Key and claimedEd25519Key.
-     *     See {@link module:models/event.MatrixEvent#getForwardingCurve25519KeyChain}.
+     * @param {module:crypto~EventDecryptionResult} decryptionResult
+     *     the decryption result, including the plaintext and some key info
      */
-    setClearData: function(
-        clearEvent,
-        senderCurve25519Key,
-        claimedEd25519Key,
-        forwardingCurve25519KeyChain,
-    ) {
-        this._clearEvent = clearEvent;
-        this._senderCurve25519Key = senderCurve25519Key || null;
-        this._claimedEd25519Key = claimedEd25519Key || null;
-        this._forwardingCurve25519KeyChain = forwardingCurve25519KeyChain || [];
+    _setClearData: function(decryptionResult) {
+        this._clearEvent = decryptionResult.clearEvent;
+        this._senderCurve25519Key =
+            decryptionResult.senderCurve25519Key || null;
+        this._claimedEd25519Key =
+            decryptionResult.claimedEd25519Key || null;
+        this._forwardingCurve25519KeyChain =
+            decryptionResult.forwardingCurve25519KeyChain || [];
         this.emit("Event.decrypted", this);
     },
 
@@ -600,7 +668,6 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         this.event = event;
         // successfully sent.
         this.status = null;
-        this._date = new Date(this.event.origin_server_ts);
      },
 });
 

src/models/group.js

@@ -0,0 +1,95 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module models/group
+ */
+const EventEmitter = require("events").EventEmitter;
+
+const utils = require("../utils");
+
+/**
+ * Construct a new Group.
+ *
+ * @param {string} groupId The ID of this group.
+ *
+ * @prop {string} groupId The ID of this group.
+ * @prop {string} name The human-readable display name for this group.
+ * @prop {string} avatarUrl The mxc URL for this group's avatar.
+ * @prop {string} myMembership The logged in user's membership of this group
+ * @prop {Object} inviter Infomation about the user who invited the logged in user
+ *       to the group, if myMembership is 'invite'.
+ * @prop {string} inviter.userId The user ID of the inviter
+ */
+function Group(groupId) {
+    this.groupId = groupId;
+    this.name = null;
+    this.avatarUrl = null;
+    this.myMembership = null;
+    this.inviter = null;
+}
+utils.inherits(Group, EventEmitter);
+
+Group.prototype.setProfile = function(name, avatarUrl) {
+    if (this.name === name && this.avatarUrl === avatarUrl) return;
+
+    this.name = name || this.groupId;
+    this.avatarUrl = avatarUrl;
+
+    this.emit("Group.profile", this);
+};
+
+Group.prototype.setMyMembership = function(membership) {
+    if (this.myMembership === membership) return;
+
+    this.myMembership = membership;
+
+    this.emit("Group.myMembership", this);
+};
+
+/**
+ * Sets the 'inviter' property. This does not emit an event (the inviter
+ * will only change when the user is revited / reinvited to a room),
+ * so set this before setting myMembership.
+ * @param {Object} inviter Infomation about who invited us to the room
+ */
+Group.prototype.setInviter = function(inviter) {
+    this.inviter = inviter;
+};
+
+module.exports = Group;
+
+/**
+ * Fires whenever a group's profile information is updated.
+ * This means the 'name' and 'avatarUrl' properties.
+ * @event module:client~MatrixClient#"Group.profile"
+ * @param {Group} group The group whose profile was updated.
+ * @example
+ * matrixClient.on("Group.profile", function(group){
+ *   var name = group.name;
+ * });
+ */
+
+/**
+ * Fires whenever the logged in user's membership status of
+ * the group is updated.
+ * @event module:client~MatrixClient#"Group.myMembership"
+ * @param {Group} group The group in which the user's membership changed
+ * @example
+ * matrixClient.on("Group.myMembership", function(group){
+ *   var myMembership = group.myMembership;
+ * });
+ */

src/models/room-member.js

@@ -104,17 +104,20 @@ RoomMember.prototype.setPowerLevelEvent = function(powerLevelEvent) {
     if (powerLevelEvent.getType() !== "m.room.power_levels") {
         return;
     }
-    let maxLevel = powerLevelEvent.getContent().users_default || 0;
-    utils.forEach(utils.values(powerLevelEvent.getContent().users), function(lvl) {
+
+    const evContent = powerLevelEvent.getDirectionalContent();
+
+    let maxLevel = evContent.users_default || 0;
+    utils.forEach(utils.values(evContent.users), function(lvl) {
         maxLevel = Math.max(maxLevel, lvl);
     });
     const oldPowerLevel = this.powerLevel;
     const oldPowerLevelNorm = this.powerLevelNorm;
 
-    if (powerLevelEvent.getContent().users[this.userId] !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users[this.userId];
-    } else if (powerLevelEvent.getContent().users_default !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users_default;
+    if (evContent.users && evContent.users[this.userId] !== undefined) {
+        this.powerLevel = evContent.users[this.userId];
+    } else if (evContent.users_default !== undefined) {
+        this.powerLevel = evContent.users_default;
     } else {
         this.powerLevel = 0;
     }
@@ -226,11 +229,21 @@ function calculateDisplayName(member, event, roomState) {
         return displayName;
     }
 
-    const userIds = roomState.getUserIdsWithDisplayName(displayName);
-    const otherUsers = userIds.filter(function(u) {
-        return u !== selfUserId;
-    });
-    if (otherUsers.length > 0) {
+
+    // Check if the name contains something that look like a mxid
+    // If it does, it may be someone trying to impersonate someone else
+    // Show full mxid in this case
+    // Also show mxid if there are other people with the same displayname
+    let disambiguate = /@.+:.+/.test(displayName);
+    if (!disambiguate) {
+        const userIds = roomState.getUserIdsWithDisplayName(displayName);
+        const otherUsers = userIds.filter(function(u) {
+            return u !== selfUserId;
+        });
+        disambiguate = otherUsers.length > 0;
+    }
+
+    if (disambiguate) {
         return displayName + " (" + selfUserId + ")";
     }
     return displayName;

src/models/room-state.js

@@ -24,6 +24,25 @@ const RoomMember = require("./room-member");
 
 /**
  * Construct room state.
+ *
+ * Room State represents the state of the room at a given point.
+ * It can be mutated by adding state events to it.
+ * There are two types of room member associated with a state event:
+ * normal member objects (accessed via getMember/getMembers) which mutate
+ * with the state to represent the current state of that room/user, eg.
+ * the object returned by getMember('@bob:example.com') will mutate to
+ * get a different display name if Bob later changes his display name
+ * in the room.
+ * There are also 'sentinel' members (accessed via getSentinelMember).
+ * These also represent the state of room members at the point in time
+ * represented by the RoomState object, but unlike objects from getMember,
+ * sentinel objects will always represent the room state as at the time
+ * getSentinelMember was called, so if Bob subsequently changes his display
+ * name, a room member object previously acquired with getSentinelMember
+ * will still have his old display name. Calling getSentinelMember again
+ * after the display name change will return a new RoomMember object
+ * with Bob's new display name.
+ *
  * @constructor
  * @param {?string} roomId Optional. The ID of the room which has this state.
  * If none is specified it just tracks paginationTokens, useful for notifTimelineSet
@@ -50,9 +69,24 @@ function RoomState(roomId) {
     this._displayNameToUserIds = {};
     this._userIdsToDisplayNames = {};
     this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite
+    this._joinedMemberCount = null; // cache of the number of joined members
 }
 utils.inherits(RoomState, EventEmitter);
 
+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+RoomState.prototype.getJoinedMemberCount = function() {
+    if (this._joinedMemberCount === null) {
+        this._joinedMemberCount = this.getMembers().filter((m) => {
+            return m.membership === 'join';
+        }).length;
+    }
+    return this._joinedMemberCount;
+};
+
 /**
  * Get all RoomMembers in this room.
  * @return {Array<RoomMember>} A list of RoomMembers.
@@ -80,7 +114,21 @@ RoomState.prototype.getMember = function(userId) {
  * @return {RoomMember} The member or null if they do not exist.
  */
 RoomState.prototype.getSentinelMember = function(userId) {
-    return this._sentinels[userId] || null;
+    if (!userId) return null;
+    let sentinel = this._sentinels[userId];
+
+    if (sentinel === undefined) {
+        sentinel = new RoomMember(this.roomId, userId);
+        const membershipEvent = this.getStateEvents("m.room.member", userId);
+        if (!membershipEvent) return null;
+        sentinel.setMembershipEvent(membershipEvent, this);
+        const pwrLvlEvent = this.getStateEvents("m.room.power_levels", "");
+        if (pwrLvlEvent) {
+            sentinel.setPowerLevelEvent(pwrLvlEvent);
+        }
+        this._sentinels[userId] = sentinel;
+    }
+    return sentinel;
 };
 
 /**
@@ -173,23 +221,19 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                 member = new RoomMember(event.getRoomId(), userId);
                 self.emit("RoomState.newMember", event, self, member);
             }
-            // Add a new sentinel for this change. We apply the same
-            // operations to both sentinel and member rather than deep copying
-            // so we don't make assumptions about the properties of RoomMember
-            // (e.g. and manage to break it because deep copying doesn't do
-            // everything).
-            const sentinel = new RoomMember(event.getRoomId(), userId);
-            utils.forEach([member, sentinel], function(roomMember) {
-                roomMember.setMembershipEvent(event, self);
-                // this member may have a power level already, so set it.
-                const pwrLvlEvent = self.getStateEvents("m.room.power_levels", "");
-                if (pwrLvlEvent) {
-                    roomMember.setPowerLevelEvent(pwrLvlEvent);
-                }
-            });
 
-            self._sentinels[userId] = sentinel;
+            member.setMembershipEvent(event, self);
+            // this member may have a power level already, so set it.
+            const pwrLvlEvent = self.getStateEvents("m.room.power_levels", "");
+            if (pwrLvlEvent) {
+                member.setPowerLevelEvent(pwrLvlEvent);
+            }
+
+            // blow away the sentinel which is now outdated
+            delete self._sentinels[userId];
+
             self.members[userId] = member;
+            self._joinedMemberCount = null;
             self.emit("RoomState.members", event, self, member);
         } else if (event.getType() === "m.room.power_levels") {
             const members = utils.values(self.members);
@@ -197,6 +241,9 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                 member.setPowerLevelEvent(event);
                 self.emit("RoomState.members", event, self, member);
             });
+
+            // assume all our sentinels are now out-of-date
+            self._sentinels = {};
         }
     });
 };
@@ -258,7 +305,11 @@ RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
     if (!member || member.membership === 'leave') return false;
 
     if (mxEvent.status || mxEvent.isRedacted()) return false;
-    if (mxEvent.getSender() === userId) return true;
+
+    // The user may have been the sender, but they can't redact their own message
+    // if redactions are blocked.
+    const canRedact = this.maySendEvent("m.room.redaction", userId);
+    if (mxEvent.getSender() === userId) return canRedact;
 
     return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
 };
@@ -383,6 +434,35 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
     return member.powerLevel >= required_level;
 };
 
+/**
+ * Returns true if the given user ID has permission to trigger notification
+ * of type `notifLevelKey`
+ * @param {string} notifLevelKey The level of notification to test (eg. 'room')
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given user ID has permission to trigger a
+ *                        notification of this type.
+ */
+RoomState.prototype.mayTriggerNotifOfType = function(notifLevelKey, userId) {
+    const member = this.getMember(userId);
+    if (!member) {
+        return false;
+    }
+
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let notifLevel = 50;
+    if (
+        powerLevelsEvent &&
+        powerLevelsEvent.getContent() &&
+        powerLevelsEvent.getContent().notifications &&
+        powerLevelsEvent.getContent().notifications[notifLevelKey]
+    ) {
+        notifLevel = powerLevelsEvent.getContent().notifications[notifLevelKey];
+    }
+
+    return member.powerLevel >= notifLevel;
+};
+
 /**
  * The RoomState class.
  */

src/models/room.js

@@ -27,7 +27,7 @@ const ContentRepo = require("../content-repo");
 const EventTimeline = require("./event-timeline");
 const EventTimelineSet = require("./event-timeline-set");
 
-import reEmit from '../reemit';
+import ReEmitter from '../ReEmitter';
 
 function synthesizeReceipt(userId, event, receiptType) {
     // console.log("synthesizing receipt for "+event.getId());
@@ -106,6 +106,8 @@ function Room(roomId, opts) {
     opts = opts || {};
     opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
 
+    this.reEmitter = new ReEmitter(this);
+
     if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
         throw new Error(
             "opts.pendingEventOrdering MUST be either 'chronological' or " +
@@ -153,7 +155,7 @@ function Room(roomId, opts) {
     // all our per-room timeline sets. the first one is the unfiltered ones;
     // the subsequent ones are the filtered ones in no particular order.
     this._timelineSets = [new EventTimelineSet(this, opts)];
-    reEmit(this, this.getUnfilteredTimelineSet(),
+    this.reEmitter.reEmit(this.getUnfilteredTimelineSet(),
            ["Room.timeline", "Room.timelineReset"]);
 
     this._fixUpLegacyTimelineFields();
@@ -167,7 +169,8 @@ function Room(roomId, opts) {
         this._pendingEventList = [];
     }
 
-    this._blacklistUnverifiedDevices = false; // read by megolm
+    // read by megolm; boolean value - null indicates "use global value"
+    this._blacklistUnverifiedDevices = null;
 }
 utils.inherits(Room, EventEmitter);
 
@@ -305,8 +308,8 @@ Room.prototype.setUnreadNotificationCount = function(type, count) {
 
 /**
  * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @param {boolean} value if true, blacklist unverified devices.
+ * @param {Boolean} value true to blacklist unverified devices, null
+ * to use the global value for this room.
  */
 Room.prototype.setBlacklistUnverifiedDevices = function(value) {
     this._blacklistUnverifiedDevices = value;
@@ -314,8 +317,8 @@ Room.prototype.setBlacklistUnverifiedDevices = function(value) {
 
 /**
  * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @return {boolean} true if blacklisting unverified devices.
+ * @return {Boolean} true if blacklisting unverified devices, null
+ * if the global value should be used for this room.
  */
 Room.prototype.getBlacklistUnverifiedDevices = function() {
     return this._blacklistUnverifiedDevices;
@@ -490,7 +493,7 @@ Room.prototype.getOrCreateFilteredTimelineSet = function(filter) {
     }
     const opts = Object.assign({ filter: filter }, this._opts);
     const timelineSet = new EventTimelineSet(this, opts);
-    reEmit(this, timelineSet, ["Room.timeline", "Room.timelineReset"]);
+    this.reEmitter.reEmit(timelineSet, ["Room.timeline", "Room.timelineReset"]);
     this._filteredTimelineSets[filter.filterId] = timelineSet;
     this._timelineSets.push(timelineSet);
 
@@ -594,7 +597,8 @@ Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
     // synthesize and inject implicit read receipts
     // Done after adding the event because otherwise the app would get a read receipt
     // pointing to an event that wasn't yet in the timeline
-    if (event.sender) {
+    // Don't synthesize RR for m.room.redaction as this causes the RR to go missing.
+    if (event.sender && event.getType() !== "m.room.redaction") {
         this.addReceipt(synthesizeReceipt(
             event.sender.userId, event, "m.read",
         ), true);
@@ -1203,13 +1207,13 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     if (myMemberEvent && myMemberEvent.content.membership == "invite") {
         if (room.currentState.getMember(myMemberEvent.sender)) {
             // extract who invited us to the room
-            return "Invite from " + room.currentState.getMember(
+            return room.currentState.getMember(
                 myMemberEvent.sender,
             ).name;
         } else if (allMembers[0].events.member) {
             // use the sender field from the invite event, although this only
             // gets us the mxid
-            return "Invite from " + myMemberEvent.sender;
+            return myMemberEvent.sender;
         } else {
             return "Room Invite";
         }
@@ -1217,6 +1221,9 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
 
 
     if (otherMembers.length === 0) {
+        const leftMembers = utils.filter(room.currentState.getMembers(), function(m) {
+            return m.userId !== userId && m.membership === "leave";
+        });
         if (allMembers.length === 1) {
             // self-chat, peeked room with 1 participant,
             // or inbound invite, or outbound 3PID invite.
@@ -1236,6 +1243,10 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
                         }
                     }
                     return name;
+                } else if (leftMembers.length === 1) {
+                    // if it was a chat with one person who's now left, it's still
+                    // notionally a chat with them
+                    return leftMembers[0].name;
                 } else {
                     return "Empty room";
                 }

src/pushprocessor.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,6 +18,8 @@ limitations under the License.
  * @module pushprocessor
  */
 
+const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
+
 /**
  * Construct a Push Processor.
  * @constructor
@@ -27,12 +30,15 @@ function PushProcessor(client) {
         return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
     };
 
-    const matchingRuleFromKindSet = function(ev, kindset, device) {
-        const rulekinds_in_order = ['override', 'content', 'room', 'sender', 'underride'];
+    const cachedGlobToRegex = {
+        // $glob: RegExp,
+    };
+
+    const matchingRuleFromKindSet = (ev, kindset, device) => {
         for (let ruleKindIndex = 0;
-                ruleKindIndex < rulekinds_in_order.length;
+                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                 ++ruleKindIndex) {
-            const kind = rulekinds_in_order[ruleKindIndex];
+            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
             const ruleset = kindset[kind];
 
             for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
@@ -46,7 +52,7 @@ function PushProcessor(client) {
                     continue;
                 }
 
-                if (ruleMatchesEvent(rawrule, ev)) {
+                if (this.ruleMatchesEvent(rawrule, ev)) {
                     rule.kind = kind;
                     return rule;
                 }
@@ -73,7 +79,7 @@ function PushProcessor(client) {
                 rawrule.conditions.push({
                     'kind': 'event_match',
                     'key': 'room_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                 });
                 break;
             case 'sender':
@@ -83,7 +89,7 @@ function PushProcessor(client) {
                 rawrule.conditions.push({
                     'kind': 'event_match',
                     'key': 'user_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                 });
                 break;
             case 'content':
@@ -106,27 +112,38 @@ function PushProcessor(client) {
         return rawrule;
     };
 
-    const ruleMatchesEvent = function(rule, ev) {
-        let ret = true;
-        for (let i = 0; i < rule.conditions.length; ++i) {
-            const cond = rule.conditions[i];
-            ret &= eventFulfillsCondition(cond, ev);
-        }
-        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
-        return ret;
-    };
-
     const eventFulfillsCondition = function(cond, ev) {
         const condition_functions = {
             "event_match": eventFulfillsEventMatchCondition,
             "device": eventFulfillsDeviceCondition,
             "contains_display_name": eventFulfillsDisplayNameCondition,
             "room_member_count": eventFulfillsRoomMemberCountCondition,
+            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
         };
         if (condition_functions[cond.kind]) {
             return condition_functions[cond.kind](cond, ev);
         }
-        return true;
+        // unknown conditions: we previously matched all unknown conditions,
+        // but given that rules can be added to the base rules on a server,
+        // it's probably better to not match unknown conditions.
+        return false;
+    };
+
+    const eventFulfillsSenderNotifPermCondition = function(cond, ev) {
+        const notifLevelKey = cond['key'];
+        if (!notifLevelKey) {
+            return false;
+        }
+
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState) {
+            return false;
+        }
+
+        // Note that this should not be the current state of the room but the state at
+        // the point the event is in the DAG. Unfortunately the js-sdk does not store
+        // this.
+        return room.currentState.mayTriggerNotifOfType(notifLevelKey, ev.getSender());
     };
 
     const eventFulfillsRoomMemberCountCondition = function(cond, ev) {
@@ -139,9 +156,7 @@ function PushProcessor(client) {
             return false;
         }
 
-        const memberCount = Object.keys(room.currentState.members).filter(function(m) {
-            return room.currentState.members[m].membership == 'join';
-        }).length;
+        const memberCount = room.currentState.getJoinedMemberCount();
 
         const m = cond.is.match(/^([=<>]*)([0-9]*)$/);
         if (!m) {
@@ -194,30 +209,50 @@ function PushProcessor(client) {
     };
 
     const eventFulfillsEventMatchCondition = function(cond, ev) {
+        if (!cond.key) {
+            return false;
+        }
+
         const val = valueForDottedKey(cond.key, ev);
         if (!val || typeof val != 'string') {
             return false;
         }
 
-        let pat;
-        if (cond.key == 'content.body') {
-            pat = '(^|\\W)' + globToRegexp(cond.pattern) + '(\\W|$)';
-        } else {
-            pat = '^' + globToRegexp(cond.pattern) + '$';
+        if (cond.value) {
+            return cond.value === val;
         }
-        const regex = new RegExp(pat, 'i');
+
+        let regex;
+
+        if (cond.key == 'content.body') {
+            regex = createCachedRegex('(^|\\W)', cond.pattern, '(\\W|$)');
+        } else {
+            regex = createCachedRegex('^', cond.pattern, '$');
+        }
+
         return !!val.match(regex);
     };
 
+    const createCachedRegex = function(prefix, glob, suffix) {
+        if (cachedGlobToRegex[glob]) {
+            return cachedGlobToRegex[glob];
+        }
+        cachedGlobToRegex[glob] = new RegExp(
+            prefix + globToRegexp(glob) + suffix,
+            'i', // Case insensitive
+        );
+        return cachedGlobToRegex[glob];
+    };
+
     const globToRegexp = function(glob) {
         // From
         // https://github.com/matrix-org/synapse/blob/abbee6b29be80a77e05730707602f3bbfc3f38cb/synapse/push/__init__.py#L132
         // Because micromatch is about 130KB with dependencies,
         // and minimatch is not much better.
         let pat = escapeRegExp(glob);
-        pat = pat.replace(/\\\*/, '.*');
-        pat = pat.replace(/\?/, '.');
-        pat = pat.replace(/\\\[(!|)(.*)\\]/, function(match, p1, p2, offset, string) {
+        pat = pat.replace(/\\\*/g, '.*');
+        pat = pat.replace(/\?/g, '.');
+        pat = pat.replace(/\\\[(!|)(.*)\\]/g, function(match, p1, p2, offset, string) {
             const first = p1 && '^' || '';
             const second = p2.replace(/\\\-/, '-');
             return '[' + first + second + ']';
@@ -291,6 +326,17 @@ function PushProcessor(client) {
         return actionObj;
     };
 
+    this.ruleMatchesEvent = function(rule, ev) {
+        let ret = true;
+        for (let i = 0; i < rule.conditions.length; ++i) {
+            const cond = rule.conditions[i];
+            ret &= eventFulfillsCondition(cond, ev);
+        }
+        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
+        return ret;
+    };
+
+
     /**
      * Get the user's push actions for the given event
      *
@@ -301,6 +347,27 @@ function PushProcessor(client) {
     this.actionsForEvent = function(ev) {
         return pushActionsForEventAndRulesets(ev, client.pushRules);
     };
+
+    /**
+     * Get one of the users push rules by its ID
+     *
+     * @param {string} ruleId The ID of the rule to search for
+     * @return {object} The push rule, or null if no such rule was found
+     */
+    this.getPushRuleById = function(ruleId) {
+        for (const scope of ['device', 'global']) {
+            if (client.pushRules[scope] === undefined) continue;
+
+            for (const kind of RULEKINDS_IN_ORDER) {
+                if (client.pushRules[scope][kind] === undefined) continue;
+
+                for (const rule of client.pushRules[scope][kind]) {
+                    if (rule.rule_id === ruleId) return rule;
+                }
+            }
+        }
+        return null;
+    };
 }
 
 /**

src/reemit.js

@@ -1,44 +0,0 @@
-/*
-Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * @module
- */
-
-/**
- * re-emit events raised by one EventEmitter from another
- *
- * @param {external:EventEmitter} reEmitEntity
- *     entity from which we want events to be emitted
- * @param {external:EventEmitter} emittableEntity
- *     entity from which events are currently emitted
- * @param {Array<string>} eventNames
- *     list of events to be reemitted
- */
-export default function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    for (const eventName of eventNames) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function(...args) {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            reEmitEntity.emit(eventName, ...args);
-        });
-    }
-}

src/store/indexeddb-local-backend.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -101,6 +102,7 @@ const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
     this.indexedDB = indexedDBInterface;
     this._dbName = "matrix-js-sdk:" + (dbName || "default");
     this.db = null;
+    this._disconnected = true;
     this._syncAccumulator = new SyncAccumulator();
 };
 
@@ -112,15 +114,17 @@ LocalIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if successfully connected.
      */
     connect: function() {
-        if (this.db) {
+        if (!this._disconnected) {
             console.log(
-                `LocalIndexedDBStoreBackend.connect: already connected`,
+                `LocalIndexedDBStoreBackend.connect: already connected or connecting`,
             );
             return Promise.resolve();
         }
 
+        this._disconnected = false;
+
         console.log(
-            `LocalIndexedDBStoreBackend.connect: connecting`,
+            `LocalIndexedDBStoreBackend.connect: connecting...`,
         );
         const req = this.indexedDB.open(this._dbName, VERSION);
         req.onupgradeneeded = (ev) => {
@@ -142,7 +146,7 @@ LocalIndexedDBStoreBackend.prototype = {
         };
 
         console.log(
-            `LocalIndexedDBStoreBackend.connect: awaiting connection`,
+            `LocalIndexedDBStoreBackend.connect: awaiting connection...`,
         );
         return promiseifyRequest(req).then((ev) => {
             console.log(
@@ -175,6 +179,7 @@ LocalIndexedDBStoreBackend.prototype = {
             this._syncAccumulator.accumulate({
                 next_batch: syncData.nextBatch,
                 rooms: syncData.roomsData,
+                groups: syncData.groupsData,
                 account_data: {
                     events: accountData,
                 },
@@ -218,7 +223,7 @@ LocalIndexedDBStoreBackend.prototype = {
 
     /**
      * @param {boolean=} copy If false, the data returned is from internal
-     * buffers and must not be muated. Otherwise, a copy is made before
+     * buffers and must not be mutated. Otherwise, a copy is made before
      * returning such that the data can be safely mutated. Default: true.
      *
      * @return {Promise} Resolves with a sync response to restore the
@@ -239,6 +244,10 @@ LocalIndexedDBStoreBackend.prototype = {
         }
     },
 
+    getNextBatchToken: function() {
+        return Promise.resolve(this._syncAccumulator.getNextBatchToken());
+    },
+
     setSyncData: function(syncData) {
         return Promise.resolve().then(() => {
             this._syncAccumulator.accumulate(syncData);
@@ -251,7 +260,9 @@ LocalIndexedDBStoreBackend.prototype = {
         return Promise.all([
             this._persistUserPresenceEvents(userTuples),
             this._persistAccountData(syncData.accountData),
-            this._persistSyncData(syncData.nextBatch, syncData.roomsData),
+            this._persistSyncData(
+                syncData.nextBatch, syncData.roomsData, syncData.groupsData,
+            ),
         ]);
     },
 
@@ -259,9 +270,10 @@ LocalIndexedDBStoreBackend.prototype = {
      * Persist rooms /sync data along with the next batch token.
      * @param {string} nextBatch The next_batch /sync value.
      * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @param {Object} groupsData The 'groups' /sync data from a SyncAccumulator
      * @return {Promise} Resolves if the data was persisted.
      */
-    _persistSyncData: function(nextBatch, roomsData) {
+    _persistSyncData: function(nextBatch, roomsData, groupsData) {
         console.log("Persisting sync data up to ", nextBatch);
         return Promise.try(() => {
             const txn = this.db.transaction(["sync"], "readwrite");
@@ -270,6 +282,7 @@ LocalIndexedDBStoreBackend.prototype = {
                 clobber: "-", // constant key so will always clobber
                 nextBatch: nextBatch,
                 roomsData: roomsData,
+                groupsData: groupsData,
             }); // put == UPSERT
             return promiseifyTxn(txn);
         });
@@ -336,16 +349,18 @@ LocalIndexedDBStoreBackend.prototype = {
      */
     _loadAccountData: function() {
         console.log(
-            `LocalIndexedDBStoreBackend: loading account data`,
+            `LocalIndexedDBStoreBackend: loading account data...`,
         );
         return Promise.try(() => {
-            console.log(
-                `LocalIndexedDBStoreBackend: loaded account data`,
-            );
             const txn = this.db.transaction(["accountData"], "readonly");
             const store = txn.objectStore("accountData");
             return selectQuery(store, undefined, (cursor) => {
                 return cursor.value;
+            }).then((result) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded account data`,
+                );
+                return result;
             });
         });
     },
@@ -356,17 +371,17 @@ LocalIndexedDBStoreBackend.prototype = {
      */
     _loadSyncData: function() {
         console.log(
-            `LocalIndexedDBStoreBackend: loaded sync data`,
+            `LocalIndexedDBStoreBackend: loading sync data...`,
         );
         return Promise.try(() => {
-            console.log(
-                `LocalIndexedDBStoreBackend: loaded sync data`,
-            );
             const txn = this.db.transaction(["sync"], "readonly");
             const store = txn.objectStore("sync");
             return selectQuery(store, undefined, (cursor) => {
                 return cursor.value;
             }).then((results) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded sync data`,
+                );
                 if (results.length > 1) {
                     console.warn("loadSyncData: More than 1 sync row found.");
                 }

src/store/indexeddb-remote-backend.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,25 +27,23 @@ import Promise from 'bluebird';
  * @param {string} workerScript URL to the worker script
  * @param {string=} dbName Optional database name. The same name must be used
  * to open the same database.
- * @param {Object} WorkerApi The web worker compatible interface object
+ * @param {Object} workerApi The web worker compatible interface object
  */
 const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
-    workerScript, dbName, WorkerApi,
+    workerScript, dbName, workerApi,
 ) {
+    this._workerScript = workerScript;
     this._dbName = dbName;
-    this._worker = new WorkerApi(workerScript);
+    this._workerApi = workerApi;
+    this._worker = null;
     this._nextSeq = 0;
     // The currently in-flight requests to the actual backend
     this._inFlight = {
         // seq: promise,
     };
-
-    this._worker.onmessage = this._onWorkerMessage.bind(this);
-
-    // tell the worker the db name.
-    this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
-        console.log("IndexedDB worker is ready");
-    });
+    // Once we start connecting, we keep the promise and re-use it
+    // if we try to connect again
+    this._startPromise = null;
 };
 
 
@@ -55,7 +54,7 @@ RemoteIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolves if successfully connected.
      */
     connect: function() {
-        return this._startPromise.then(() => this._doCmd('connect'));
+        return this._ensureStarted().then(() => this._doCmd('connect'));
     },
 
     /**
@@ -64,7 +63,7 @@ RemoteIndexedDBStoreBackend.prototype = {
      * @return {Promise} Resolved when the database is cleared.
      */
     clearDatabase: function() {
-        return this._startPromise.then(() => this._doCmd('clearDatabase'));
+        return this._ensureStarted().then(() => this._doCmd('clearDatabase'));
     },
 
     /**
@@ -76,6 +75,10 @@ RemoteIndexedDBStoreBackend.prototype = {
         return this._doCmd('getSavedSync');
     },
 
+    getNextBatchToken: function() {
+        return this._doCmd('getNextBatchToken');
+    },
+
     setSyncData: function(syncData) {
         return this._doCmd('setSyncData', [syncData]);
     },
@@ -93,6 +96,19 @@ RemoteIndexedDBStoreBackend.prototype = {
         return this._doCmd('getUserPresenceEvents');
     },
 
+    _ensureStarted: function() {
+        if (this._startPromise === null) {
+            this._worker = new this._workerApi(this._workerScript);
+            this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+            // tell the worker the db name.
+            this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
+                console.log("IndexedDB worker is ready");
+            });
+        }
+        return this._startPromise;
+    },
+
     _doCmd: function(cmd, args) {
         // wrap in a q so if the postMessage throws,
         // the promise automatically gets rejected

src/store/indexeddb-store-worker.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -88,6 +89,9 @@ class IndexedDBStoreWorker {
             case 'getUserPresenceEvents':
                 prom = this.backend.getUserPresenceEvents();
                 break;
+            case 'getNextBatchToken':
+                prom = this.backend.getNextBatchToken();
+                break;
         }
 
         if (prom === undefined) {

src/store/indexeddb.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -145,6 +146,14 @@ IndexedDBStore.prototype.getSavedSync = function() {
     return this.backend.getSavedSync();
 };
 
+/**
+ * @return {Promise} If there is a saved sync, the nextBatch token
+ * for this sync, otherwise null.
+ */
+IndexedDBStore.prototype.getSavedSyncToken = function() {
+    return this.backend.getNextBatchToken();
+},
+
 /**
  * Delete all data from this store.
  * @return {Promise} Resolves if the data was deleted from the database.
@@ -159,13 +168,27 @@ IndexedDBStore.prototype.deleteAllData = function() {
     });
 };
 
+/**
+ * Whether this store would like to save its data
+ * Note that obviously whether the store wants to save or
+ * not could change between calling this function and calling
+ * save().
+ *
+ * @return {boolean} True if calling save() will actually save
+ *     (at the time this function is called).
+ */
+IndexedDBStore.prototype.wantsSave = function() {
+    const now = Date.now();
+    return now - this._syncTs > WRITE_DELAY_MS;
+};
+
 /**
  * Possibly write data to the database.
- * @return {Promise} Promise resolves after the write completes.
+ * @return {Promise} Promise resolves after the write completes
+ *     (or immediately if no write is performed)
  */
 IndexedDBStore.prototype.save = function() {
-    const now = Date.now();
-    if (now - this._syncTs > WRITE_DELAY_MS) {
+    if (this.wantsSave()) {
         return this._reallySave();
     }
     return Promise.resolve();

src/store/memory.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,8 +20,8 @@ limitations under the License.
  * This is an internal module. See {@link MatrixInMemoryStore} for the public class.
  * @module store/memory
  */
- const utils = require("../utils");
- const User = require("../models/user");
+const utils = require("../utils");
+const User = require("../models/user");
 import Promise from 'bluebird';
 
 /**
@@ -35,6 +36,9 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
     this.rooms = {
         // roomId: Room
     };
+    this.groups = {
+        // groupId: Group
+    };
     this.users = {
         // userId: User
     };
@@ -69,6 +73,31 @@ module.exports.MatrixInMemoryStore.prototype = {
         this.syncToken = token;
     },
 
+    /**
+     * Store the given room.
+     * @param {Group} group The group to be stored
+     */
+    storeGroup: function(group) {
+        this.groups[group.groupId] = group;
+    },
+
+    /**
+     * Retrieve a group by its group ID.
+     * @param {string} groupId The group ID.
+     * @return {Group} The group or null.
+     */
+    getGroup: function(groupId) {
+        return this.groups[groupId] || null;
+    },
+
+    /**
+     * Retrieve all known groups.
+     * @return {Group[]} A list of groups, which may be empty.
+     */
+    getGroups: function() {
+        return utils.values(this.groups);
+    },
+
     /**
      * Store the given room.
      * @param {Room} room The room to be stored. All properties must be stored.
@@ -287,6 +316,15 @@ module.exports.MatrixInMemoryStore.prototype = {
         return Promise.resolve();
     },
 
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
     /**
      * Save does nothing as there is no backing data store.
      */
@@ -309,6 +347,14 @@ module.exports.MatrixInMemoryStore.prototype = {
         return Promise.resolve(null);
     },
 
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
     /**
      * Delete all data from this store.
      * @return {Promise} An immediately resolved promise.

src/store/session/webstorage.js

@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -48,17 +50,18 @@ function WebStorageSessionStore(webStore) {
 }
 
 WebStorageSessionStore.prototype = {
-
     /**
-     * Store the end to end account for the logged-in user.
-     * @param {string} account Base64 encoded account.
+     * Remove the stored end to end account for the logged-in user.
      */
-    storeEndToEndAccount: function(account) {
-        this.store.setItem(KEY_END_TO_END_ACCOUNT, account);
+    removeEndToEndAccount: function() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
     },
 
     /**
      * Load the end to end account for the logged-in user.
+     * Note that the end-to-end account is now stored in the
+     * crypto store rather than here: this remains here so
+     * old sessions can be migrated out of the session store.
      * @return {?string} Base64 encoded account.
      */
     getEndToEndAccount: function() {
@@ -66,43 +69,24 @@ WebStorageSessionStore.prototype = {
     },
 
     /**
-     * Stores the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @param {object} devices A map from device ID to keys for the device.
+     * Retrieves the known devices for all users.
+     * @return {object} A map from user ID to map of device ID to keys for the device.
      */
-    storeEndToEndDevicesForUser: function(userId, devices) {
-        setJsonItem(this.store, keyEndToEndDevicesForUser(userId), devices);
-    },
-
-    /**
-     * Retrieves the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @return {object} A map from device ID to keys for the device.
-     */
-    getEndToEndDevicesForUser: function(userId) {
-        return getJsonItem(this.store, keyEndToEndDevicesForUser(userId));
-    },
-
-    storeEndToEndDeviceTrackingStatus: function(statusMap) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS, statusMap);
+    getAllEndToEndDevices: function() {
+        const prefix = keyEndToEndDevicesForUser('');
+        const devices = {};
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            const userId = key.substr(prefix.length);
+            if (key.startsWith(prefix)) devices[userId] = getJsonItem(this.store, key);
+        }
+        return devices;
     },
 
     getEndToEndDeviceTrackingStatus: function() {
         return getJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
     },
 
-    /**
-     * Store the sync token corresponding to the device list.
-     *
-     * This is used when starting the client, to get a list of the users who
-     * have changed their device list since the list time we were running.
-     *
-     * @param {String?} token
-     */
-    storeEndToEndDeviceSyncToken: function(token) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN, token);
-    },
-
     /**
      * Get the sync token corresponding to the device list.
      *
@@ -113,17 +97,12 @@ WebStorageSessionStore.prototype = {
     },
 
     /**
-     * Store a session between the logged-in user and another device
-     * @param {string} deviceKey The public key of the other device.
-     * @param {string} sessionId The ID for this end-to-end session.
-     * @param {string} session Base64 encoded end-to-end session.
+     * Removes all end to end device data from the store
      */
-    storeEndToEndSession: function(deviceKey, sessionId, session) {
-        const sessions = this.getEndToEndSessions(deviceKey) || {};
-        sessions[sessionId] = session;
-        setJsonItem(
-            this.store, keyEndToEndSessions(deviceKey), sessions,
-        );
+    removeEndToEndDeviceData: function() {
+        removeByPrefix(this.store, keyEndToEndDevicesForUser(''));
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN);
     },
 
     /**
@@ -136,6 +115,29 @@ WebStorageSessionStore.prototype = {
         return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
     },
 
+    /**
+     * Retrieve all end-to-end sessions between the logged-in user and other
+     * devices.
+     * @return {object} A map of {deviceKey -> {sessionId -> session pickle}}
+     */
+    getAllEndToEndSessions: function() {
+        const deviceKeys = getKeysWithPrefix(this.store, keyEndToEndSessions(''));
+        const results = {};
+        for (const k of deviceKeys) {
+            const unprefixedKey = k.substr(keyEndToEndSessions('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
+    },
+
+    /**
+     * Remove all end-to-end sessions from the store
+     * This is used after migrating sessions awat from the sessions store.
+     */
+    removeAllEndToEndSessions: function() {
+        removeByPrefix(this.store, keyEndToEndSessions(''));
+    },
+
     /**
      * Retrieve a list of all known inbound group sessions
      *
@@ -167,27 +169,26 @@ WebStorageSessionStore.prototype = {
         return this.store.getItem(key);
     },
 
-    storeEndToEndInboundGroupSession: function(senderKey, sessionId, pickledSession) {
-        const key = keyEndToEndInboundGroupSession(senderKey, sessionId);
-        return this.store.setItem(key, pickledSession);
+    removeAllEndToEndInboundGroupSessions: function() {
+        removeByPrefix(this.store, E2E_PREFIX + 'inboundgroupsessions/');
     },
 
     /**
-     * Store the end-to-end state for a room.
-     * @param {string} roomId The room's ID.
-     * @param {object} roomInfo The end-to-end info for the room.
+     * Get the end-to-end state for all rooms
+     * @return {object} roomId -> object with the end-to-end info for the room.
      */
-    storeEndToEndRoom: function(roomId, roomInfo) {
-        setJsonItem(this.store, keyEndToEndRoom(roomId), roomInfo);
+    getAllEndToEndRooms: function() {
+        const roomKeys = getKeysWithPrefix(this.store, keyEndToEndRoom(''));
+        const results = {};
+        for (const k of roomKeys) {
+            const unprefixedKey = k.substr(keyEndToEndRoom('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
     },
 
-    /**
-     * Get the end-to-end state for a room
-     * @param {string} roomId The room's ID.
-     * @return {object} The end-to-end info for the room.
-     */
-    getEndToEndRoom: function(roomId) {
-        return getJsonItem(this.store, keyEndToEndRoom(roomId));
+    removeAllEndToEndRooms: function() {
+        removeByPrefix(this.store, keyEndToEndRoom(''));
     },
 };
 
@@ -223,8 +224,24 @@ function getJsonItem(store, key) {
     return null;
 }
 
-function setJsonItem(store, key, val) {
-    store.setItem(key, JSON.stringify(val));
+function getKeysWithPrefix(store, prefix) {
+    const results = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) results.push(key);
+    }
+    return results;
+}
+
+function removeByPrefix(store, prefix) {
+    const toRemove = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) toRemove.push(key);
+    }
+    for (const key of toRemove) {
+        store.removeItem(key);
+    }
 }
 
 function debuglog() {

src/store/stub.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -47,6 +48,30 @@ StubStore.prototype = {
         this.fromToken = token;
     },
 
+    /**
+     * No-op.
+     * @param {Group} group
+     */
+    storeGroup: function(group) {
+    },
+
+    /**
+     * No-op.
+     * @param {string} groupId
+     * @return {null}
+     */
+    getGroup: function(groupId) {
+        return null;
+    },
+
+    /**
+     * No-op.
+     * @return {Array} An empty array.
+     */
+    getGroups: function() {
+        return [];
+    },
+
     /**
      * No-op.
      * @param {Room} room
@@ -192,6 +217,15 @@ StubStore.prototype = {
         return Promise.resolve();
     },
 
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
     /**
      * Save does nothing as there is no backing data store.
      */
@@ -214,6 +248,14 @@ StubStore.prototype = {
         return Promise.resolve(null);
     },
 
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
     /**
      * Delete all data from this store. Does nothing since this store
      * doesn't store anything.

src/sync-accumulator.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -72,10 +73,18 @@ class SyncAccumulator {
         // coherent /sync response and know at what point they should be
         // streaming from without losing events.
         this.nextBatch = null;
+
+        // { ('invite'|'join'|'leave'): $groupId: { ... sync 'group' data } }
+        this.groups = {
+            invite: {},
+            join: {},
+            leave: {},
+        };
     }
 
     accumulate(syncResponse) {
         this._accumulateRooms(syncResponse);
+        this._accumulateGroups(syncResponse);
         this._accumulateAccountData(syncResponse);
         this.nextBatch = syncResponse.next_batch;
     }
@@ -336,6 +345,44 @@ class SyncAccumulator {
         }
     }
 
+    /**
+     * Accumulate incremental /sync group data.
+     * @param {Object} syncResponse the complete /sync JSON
+     */
+    _accumulateGroups(syncResponse) {
+        if (!syncResponse.groups) {
+            return;
+        }
+        if (syncResponse.groups.invite) {
+            Object.keys(syncResponse.groups.invite).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "invite", syncResponse.groups.invite[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.join) {
+            Object.keys(syncResponse.groups.join).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "join", syncResponse.groups.join[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.leave) {
+            Object.keys(syncResponse.groups.leave).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "leave", syncResponse.groups.leave[groupId],
+                );
+            });
+        }
+    }
+
+    _accumulateGroup(groupId, category, data) {
+        for (const cat of ['invite', 'join', 'leave']) {
+            delete this.groups[cat][groupId];
+        }
+        this.groups[category][groupId] = data;
+    }
+
     /**
      * Return everything under the 'rooms' key from a /sync response which
      * represents all room data that should be stored. This should be paired
@@ -470,9 +517,14 @@ class SyncAccumulator {
         return {
             nextBatch: this.nextBatch,
             roomsData: data,
+            groupsData: this.groups,
             accountData: accData,
         };
     }
+
+    getNextBatchToken() {
+        return this.nextBatch;
+    }
 }
 
 function setState(eventMap, event) {

src/sync.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -27,12 +28,11 @@ limitations under the License.
 import Promise from 'bluebird';
 const User = require("./models/user");
 const Room = require("./models/room");
+const Group = require('./models/group');
 const utils = require("./utils");
 const Filter = require("./filter");
 const EventTimeline = require("./models/event-timeline");
 
-import reEmit from './reemit';
-
 const DEBUG = true;
 
 // /sync requests allow you to set a timeout= but the request may continue
@@ -72,6 +72,8 @@ function debuglog(...params) {
  * SAFELY remove events from this room. It may not be safe to remove events if
  * there are other references to the timelines for this room.
  * Default: returns false.
+ * @param {Boolean=} opts.disablePresence True to perform syncing without automatically
+ * updating presence.
  */
 function SyncApi(client, opts) {
     this.client = client;
@@ -99,7 +101,7 @@ function SyncApi(client, opts) {
     this._failedSyncCount = 0; // Number of consecutive failed /sync requests
 
     if (client.getNotifTimelineSet()) {
-        reEmit(client, client.getNotifTimelineSet(),
+        client.reEmitter.reEmit(client.getNotifTimelineSet(),
                ["Room.timeline", "Room.timelineReset"]);
     }
 }
@@ -114,7 +116,7 @@ SyncApi.prototype.createRoom = function(roomId) {
         pendingEventOrdering: this.opts.pendingEventOrdering,
         timelineSupport: client.timelineSupport,
     });
-    reEmit(client, room, ["Room.name", "Room.timeline", "Room.redaction",
+    client.reEmitter.reEmit(room, ["Room.name", "Room.timeline", "Room.redaction",
                           "Room.receipt", "Room.tags",
                           "Room.timelineReset",
                           "Room.localEchoUpdated",
@@ -124,6 +126,18 @@ SyncApi.prototype.createRoom = function(roomId) {
     return room;
 };
 
+/**
+ * @param {string} groupId
+ * @return {Group}
+ */
+SyncApi.prototype.createGroup = function(groupId) {
+    const client = this.client;
+    const group = new Group(groupId);
+    client.reEmitter.reEmit(group, ["Group.profile", "Group.myMembership"]);
+    client.store.storeGroup(group);
+    return group;
+};
+
 /**
  * @param {Room} room
  * @private
@@ -133,13 +147,13 @@ SyncApi.prototype._registerStateListeners = function(room) {
     // we need to also re-emit room state and room member events, so hook it up
     // to the client now. We need to add a listener for RoomState.members in
     // order to hook them correctly. (TODO: find a better way?)
-    reEmit(client, room.currentState, [
+    client.reEmitter.reEmit(room.currentState, [
         "RoomState.events", "RoomState.members", "RoomState.newMember",
     ]);
     room.currentState.on("RoomState.newMember", function(event, state, member) {
         member.user = client.getUser(member.userId);
-        reEmit(
-            client, member,
+        client.reEmitter.reEmit(
+            member,
             [
                 "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel",
                 "RoomMember.membership",
@@ -220,6 +234,8 @@ SyncApi.prototype.syncLeftRooms = function() {
             room.recalculate(client.credentials.userId);
             client.store.storeRoom(room);
             client.emit("Room", room);
+
+            self._processEventsForNotifs(room, timelineEvents);
         });
         return rooms;
     });
@@ -380,6 +396,17 @@ SyncApi.prototype.getSyncState = function() {
     return this._syncState;
 };
 
+SyncApi.prototype.recoverFromSyncStartupError = async function(savedSyncPromise, err) {
+    // Wait for the saved sync to complete - we send the pushrules and filter requests
+    // before the saved sync has finished so they can run in parallel, but only process
+    // the results after the saved sync is done. Equivalently, we wait for it to finish
+    // before reporting failures from these functions.
+    await savedSyncPromise;
+    const keepaliveProm = this._startKeepAlives();
+    this._updateSyncState("ERROR", { error: err });
+    await keepaliveProm;
+};
+
 /**
  * Main entry point
  */
@@ -394,26 +421,32 @@ SyncApi.prototype.sync = function() {
         global.document.addEventListener("online", this._onOnlineBound, false);
     }
 
+    let savedSyncPromise = Promise.resolve();
+    let savedSyncToken = null;
+
     // We need to do one-off checks before we can begin the /sync loop.
     // These are:
     //   1) We need to get push rules so we can check if events should bing as we get
     //      them from /sync.
     //   2) We need to get/create a filter which we can use for /sync.
 
-    function getPushRules() {
-        client.getPushRules().done(function(result) {
+    async function getPushRules() {
+        try {
+            const result = await client.getPushRules();
             debuglog("Got push rules");
+
             client.pushRules = result;
-            getFilter(); // Now get the filter
-        }, function(err) {
-            self._startKeepAlives().done(function() {
-                getPushRules();
-            });
-            self._updateSyncState("ERROR", { error: err });
-        });
+        } catch (err) {
+            // wait for saved sync to complete before doing anything else,
+            // otherwise the sync state will end up being incorrect
+            await self.recoverFromSyncStartupError(savedSyncPromise, err);
+            getPushRules();
+            return;
+        }
+        getFilter(); // Now get the filter and start syncing
     }
 
-    function getFilter() {
+    async function getFilter() {
         let filter;
         if (self.opts.filter) {
             filter = self.opts.filter;
@@ -422,28 +455,54 @@ SyncApi.prototype.sync = function() {
             filter.setTimelineLimit(self.opts.initialSyncLimit);
         }
 
-        client.getOrCreateFilter(
-            getFilterName(client.credentials.userId), filter,
-        ).done(function(filterId) {
-            // reset the notifications timeline to prepare it to paginate from
-            // the current point in time.
-            // The right solution would be to tie /sync pagination tokens into
-            // /notifications API somehow.
-            client.resetNotifTimelineSet();
+        let filterId;
+        try {
+            filterId = await client.getOrCreateFilter(
+                getFilterName(client.credentials.userId), filter,
+            );
+        } catch (err) {
+            // wait for saved sync to complete before doing anything else,
+            // otherwise the sync state will end up being incorrect
+            await self.recoverFromSyncStartupError(savedSyncPromise, err);
+            getFilter();
+            return;
+        }
+        // reset the notifications timeline to prepare it to paginate from
+        // the current point in time.
+        // The right solution would be to tie /sync pagination tokens into
+        // /notifications API somehow.
+        client.resetNotifTimelineSet();
 
-            self._sync({ filterId: filterId });
-        }, function(err) {
-            self._startKeepAlives().done(function() {
-                getFilter();
-            });
-            self._updateSyncState("ERROR", { error: err });
-        });
+        if (self._currentSyncRequest === null) {
+            // Send this first sync request here so we can then wait for the saved
+            // sync data to finish processing before we process the results of this one.
+            console.log("Sending first sync request...");
+            self._currentSyncRequest = self._doSyncRequest({ filterId }, savedSyncToken);
+        }
+
+        // Now wait for the saved sync to finish...
+        await savedSyncPromise;
+        self._sync({ filterId });
     }
 
     if (client.isGuest()) {
         // no push rules for guests, no access to POST filter for guests.
         self._sync({});
     } else {
+        // Pull the saved sync token out first, before the worker starts sending
+        // all the sync data which could take a while. This will let us send our
+        // first incremental sync request before we've processed our saved data.
+        savedSyncPromise = client.store.getSavedSyncToken().then((tok) => {
+            savedSyncToken = tok;
+            return client.store.getSavedSync();
+        }).then((savedSync) => {
+            if (savedSync) {
+                return self._syncFromCache(savedSync);
+            }
+        });
+        // Now start the first incremental sync request: this can also
+        // take a while so if we set it going now, we can wait for it
+        // to finish while we process our saved sync data.
         getPushRules();
     }
 };
@@ -479,6 +538,43 @@ SyncApi.prototype.retryImmediately = function() {
     this._startKeepAlives(0);
     return true;
 };
+/**
+ * Process a single set of cached sync data.
+ * @param {Object} savedSync a saved sync that was persisted by a store. This
+ * should have been acquired via client.store.getSavedSync().
+ */
+SyncApi.prototype._syncFromCache = async function(savedSync) {
+    debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
+
+    const nextSyncToken = savedSync.nextBatch;
+
+    // Set sync token for future incremental syncing
+    this.client.store.setSyncToken(nextSyncToken);
+
+    // No previous sync, set old token to null
+    const syncEventData = {
+        oldSyncToken: null,
+        nextSyncToken,
+        catchingUp: false,
+    };
+
+    const data = {
+        next_batch: nextSyncToken,
+        rooms: savedSync.roomsData,
+        groups: savedSync.groupsData,
+        account_data: {
+            events: savedSync.accountData,
+        },
+    };
+
+    try {
+        await this._processSyncResponse(syncEventData, data);
+    } catch(e) {
+        console.error("Error processing cached sync", e.stack || e);
+    }
+
+    this._updateSyncState("PREPARED", syncEventData);
+};
 
 /**
  * Invoke me to do /sync calls
@@ -486,27 +582,112 @@ SyncApi.prototype.retryImmediately = function() {
  * @param {string} syncOptions.filterId
  * @param {boolean} syncOptions.hasSyncedBefore
  */
-SyncApi.prototype._sync = function(syncOptions) {
+SyncApi.prototype._sync = async function(syncOptions) {
     const client = this.client;
-    const self = this;
 
     if (!this._running) {
         debuglog("Sync no longer running: exiting.");
-        if (self._connectionReturnedDefer) {
-            self._connectionReturnedDefer.reject();
-            self._connectionReturnedDefer = null;
+        if (this._connectionReturnedDefer) {
+            this._connectionReturnedDefer.reject();
+            this._connectionReturnedDefer = null;
         }
         this._updateSyncState("STOPPED");
         return;
     }
 
-    let filterId = syncOptions.filterId;
-    if (client.isGuest() && !filterId) {
-        filterId = this._getGuestFilter();
-    }
-
     const syncToken = client.store.getSyncToken();
 
+    let data;
+    try {
+        //debuglog('Starting sync since=' + syncToken);
+        if (this._currentSyncRequest === null) {
+            this._currentSyncRequest = this._doSyncRequest(syncOptions, syncToken);
+        }
+        data = await this._currentSyncRequest;
+    } catch (e) {
+        this._onSyncError(e, syncOptions);
+        return;
+    } finally {
+        this._currentSyncRequest = null;
+    }
+
+    //debuglog('Completed sync, next_batch=' + data.next_batch);
+
+    // set the sync token NOW *before* processing the events. We do this so
+    // if something barfs on an event we can skip it rather than constantly
+    // polling with the same token.
+    client.store.setSyncToken(data.next_batch);
+
+    // Reset after a successful sync
+    this._failedSyncCount = 0;
+
+    await client.store.setSyncData(data);
+
+    const syncEventData = {
+        oldSyncToken: syncToken,
+        nextSyncToken: data.next_batch,
+        catchingUp: this._catchingUp,
+    };
+
+    if (this.opts.crypto) {
+        // tell the crypto module we're about to process a sync
+        // response
+        await this.opts.crypto.onSyncWillProcess(syncEventData);
+    }
+
+    try {
+        await this._processSyncResponse(syncEventData, data);
+    } catch(e) {
+        // log the exception with stack if we have it, else fall back
+        // to the plain description
+        console.error("Caught /sync error", e.stack || e);
+    }
+
+    // update this as it may have changed
+    syncEventData.catchingUp = this._catchingUp;
+
+    // emit synced events
+    if (!syncOptions.hasSyncedBefore) {
+        this._updateSyncState("PREPARED", syncEventData);
+        syncOptions.hasSyncedBefore = true;
+    }
+
+    // tell the crypto module to do its processing. It may block (to do a
+    // /keys/changes request).
+    if (this.opts.crypto) {
+        await this.opts.crypto.onSyncCompleted(syncEventData);
+    }
+
+    // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
+    this._updateSyncState("SYNCING", syncEventData);
+
+    if (client.store.wantsSave()) {
+        // We always save the device list (if it's dirty) before saving the sync data:
+        // this means we know the saved device list data is at least as fresh as the
+        // stored sync data which means we don't have to worry that we may have missed
+        // device changes. We can also skip the delay since we're not calling this very
+        // frequently (and we don't really want to delay the sync for it).
+        if (this.opts.crypto) {
+            await this.opts.crypto.saveDeviceList(0);
+        }
+
+        // tell databases that everything is now in a consistent state and can be saved.
+        client.store.save();
+    }
+
+    // Begin next sync
+    this._sync(syncOptions);
+};
+
+SyncApi.prototype._doSyncRequest = function(syncOptions, syncToken) {
+    const qps = this._getSyncParams(syncOptions, syncToken);
+    return this.client._http.authedRequest(
+        undefined, "GET", "/sync", qps, undefined,
+        qps.timeout + BUFFER_PERIOD_MS,
+    );
+};
+
+SyncApi.prototype._getSyncParams = function(syncOptions, syncToken) {
     let pollTimeout = this.opts.pollTimeout;
 
     if (this.getSyncState() !== 'SYNCING' || this._catchingUp) {
@@ -525,14 +706,20 @@ SyncApi.prototype._sync = function(syncOptions) {
         pollTimeout = 0;
     }
 
-    // normal timeout= plus buffer time
-    const clientSideTimeoutMs = pollTimeout + BUFFER_PERIOD_MS;
+    let filterId = syncOptions.filterId;
+    if (this.client.isGuest() && !filterId) {
+        filterId = this._getGuestFilter();
+    }
 
     const qps = {
         filter: filterId,
         timeout: pollTimeout,
     };
 
+    if (this.opts.disablePresence) {
+        qps.set_presence = "offline";
+    }
+
     if (syncToken) {
         qps.since = syncToken;
     } else {
@@ -550,134 +737,56 @@ SyncApi.prototype._sync = function(syncOptions) {
         qps.timeout = 0;
     }
 
-    let isCachedResponse = false;
+    return qps;
+};
 
-    let syncPromise;
-    if (!syncOptions.hasSyncedBefore) {
-        // Don't do an HTTP hit to /sync. Instead, load up the persisted /sync data,
-        // if there is data there.
-        syncPromise = client.store.getSavedSync();
-    } else {
-        syncPromise = Promise.resolve(null);
+SyncApi.prototype._onSyncError = function(err, syncOptions) {
+    if (!this._running) {
+        debuglog("Sync no longer running: exiting");
+        if (this._connectionReturnedDefer) {
+            this._connectionReturnedDefer.reject();
+            this._connectionReturnedDefer = null;
+        }
+        this._updateSyncState("STOPPED");
+        return;
     }
 
-    syncPromise.then((savedSync) => {
-        if (savedSync) {
-            debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
-            isCachedResponse = true;
-            return {
-                next_batch: savedSync.nextBatch,
-                rooms: savedSync.roomsData,
-                account_data: {
-                    events: savedSync.accountData,
-                },
-            };
-        } else {
-            //debuglog('Starting sync since=' + syncToken);
-            this._currentSyncRequest = client._http.authedRequest(
-                undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
-            );
-            return this._currentSyncRequest;
-        }
-    }).then(function(data) {
-        //debuglog('Completed sync, next_batch=' + data.next_batch);
+    console.error("/sync error %s", err);
+    console.error(err);
 
-        // set the sync token NOW *before* processing the events. We do this so
-        // if something barfs on an event we can skip it rather than constantly
-        // polling with the same token.
-        client.store.setSyncToken(data.next_batch);
+    this._failedSyncCount++;
+    console.log('Number of consecutive failed sync requests:', this._failedSyncCount);
 
-        // Reset after a successful sync
-        self._failedSyncCount = 0;
-
-        // We need to wait until the sync data has been sent to the backend
-        // because it appears that the sync data gets modified somewhere in
-        // processing it in such a way as to make it no longer cloneable.
-        // XXX: Find out what is modifying it!
-        if (!isCachedResponse) {
-            // Don't give the store back its own cached data
-            return client.store.setSyncData(data).then(() => {
-                return data;
-            });
-        } else {
-            return Promise.resolve(data);
-        }
-    }).done((data) => {
-        self._processSyncResponse(syncToken, data).catch((e) => {
-            // log the exception with stack if we have it, else fall back
-            // to the plain description
-            console.error("Caught /sync error", e.stack || e);
-        }).then(() => {
-            // emit synced events
-            const syncEventData = {
-                oldSyncToken: syncToken,
-                nextSyncToken: data.next_batch,
-                catchingUp: self._catchingUp,
-            };
-
-            if (!syncOptions.hasSyncedBefore) {
-                self._updateSyncState("PREPARED", syncEventData);
-                syncOptions.hasSyncedBefore = true;
-            }
-
-            // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
-            if (!isCachedResponse) {
-                self._updateSyncState("SYNCING", syncEventData);
-
-                // tell databases that everything is now in a consistent state and can be
-                // saved (no point doing so if we only have the data we just got out of the
-                // store).
-                client.store.save();
-            }
-
-            // Begin next sync
-            self._sync(syncOptions);
-        });
-    }, function(err) {
-        if (!self._running) {
-            debuglog("Sync no longer running: exiting");
-            if (self._connectionReturnedDefer) {
-                self._connectionReturnedDefer.reject();
-                self._connectionReturnedDefer = null;
-            }
-            self._updateSyncState("STOPPED");
-            return;
-        }
-        console.error("/sync error %s", err);
-        console.error(err);
-
-        self._failedSyncCount++;
-        console.log('Number of consecutive failed sync requests:', self._failedSyncCount);
-
-        debuglog("Starting keep-alive");
-        // Note that we do *not* mark the sync connection as
-        // lost yet: we only do this if a keepalive poke
-        // fails, since long lived HTTP connections will
-        // go away sometimes and we shouldn't treat this as
-        // erroneous. We set the state to 'reconnecting'
-        // instead, so that clients can onserve this state
-        // if they wish.
-        self._startKeepAlives().done(function() {
-            self._sync(syncOptions);
-        });
-        self._currentSyncRequest = null;
-        // Transition from RECONNECTING to ERROR after a given number of failed syncs
-        self._updateSyncState(
-            self._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
-                "ERROR" : "RECONNECTING",
-        );
+    debuglog("Starting keep-alive");
+    // Note that we do *not* mark the sync connection as
+    // lost yet: we only do this if a keepalive poke
+    // fails, since long lived HTTP connections will
+    // go away sometimes and we shouldn't treat this as
+    // erroneous. We set the state to 'reconnecting'
+    // instead, so that clients can onserve this state
+    // if they wish.
+    this._startKeepAlives().then(() => {
+        this._sync(syncOptions);
     });
+
+    this._currentSyncRequest = null;
+    // Transition from RECONNECTING to ERROR after a given number of failed syncs
+    this._updateSyncState(
+        this._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
+            "ERROR" : "RECONNECTING",
+    );
 };
 
 /**
  * Process data returned from a sync response and propagate it
  * into the model objects
  *
- * @param {string} syncToken the old next_batch token sent to this
- *    sync request.
+ * @param {Object} syncEventData Object containing sync tokens associated with this sync
  * @param {Object} data The response from /sync
  */
-SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
+SyncApi.prototype._processSyncResponse = async function(
+    syncEventData, data,
+) {
     const client = this.client;
     const self = this;
 
@@ -714,6 +823,19 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
     //        }
     //      }
     //    },
+    //    groups: {
+    //        invite: {
+    //            $groupId: {
+    //                inviter: $inviter,
+    //                profile: {
+    //                    avatar_url: $avatarUrl,
+    //                    name: $groupName,
+    //                },
+    //            },
+    //        },
+    //        join: {},
+    //        leave: {},
+    //    },
     // }
 
     // TODO-arch:
@@ -743,6 +865,10 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
         client.store.storeAccountDataEvents(events);
         events.forEach(
             function(accountDataEvent) {
+                // Honour push rules that come down the sync stream but also
+                // honour push rules that were previously cached. Base rules
+                // will be updated when we recieve push rules via getPushRules
+                // (see SyncApi.prototype.sync) before syncing over the network.
                 if (accountDataEvent.getType() == 'm.push_rules') {
                     client.pushRules = accountDataEvent.getContent();
                 }
@@ -781,6 +907,20 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
         this._catchingUp = false;
     }
 
+    if (data.groups) {
+        if (data.groups.invite) {
+            this._processGroupSyncEntry(data.groups.invite, 'invite');
+        }
+
+        if (data.groups.join) {
+            this._processGroupSyncEntry(data.groups.join, 'join');
+        }
+
+        if (data.groups.leave) {
+            this._processGroupSyncEntry(data.groups.leave, 'leave');
+        }
+    }
+
     // the returned json structure is a bit crap, so make it into a
     // nicer form (array) after applying sanity to make sure we don't fail
     // on missing keys (on the off chance)
@@ -885,7 +1025,8 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
                 self._deregisterStateListeners(room);
                 room.resetLiveTimeline(
                     joinObj.timeline.prev_batch,
-                    self.opts.canResetEntireTimeline(room.roomId) ? null : syncToken,
+                    self.opts.canResetEntireTimeline(room.roomId) ?
+                        null : syncEventData.oldSyncToken,
                 );
 
                 // We have to assume any gap in any timeline is
@@ -913,6 +1054,8 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
             client.emit("Room", room);
         }
 
+        self._processEventsForNotifs(room, timelineEvents);
+
         async function processRoomEvent(e) {
             client.emit("event", e);
             if (e.isState() && e.getType() == "m.room.encryption" && self.opts.crypto) {
@@ -949,6 +1092,8 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
             client.emit("Room", room);
         }
 
+        self._processEventsForNotifs(room, timelineEvents);
+
         stateEvents.forEach(function(e) {
             client.emit("event", e);
         });
@@ -965,7 +1110,7 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
     // in the timeline relative to ones paginated in by /notifications.
     // XXX: we could fix this by making EventTimeline support chronological
     // ordering... but it doesn't, right now.
-    if (syncToken && this._notifEvents.length) {
+    if (syncEventData.oldSyncToken && this._notifEvents.length) {
         this._notifEvents.sort(function(a, b) {
             return a.getTs() - b.getTs();
         });
@@ -975,10 +1120,16 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
     }
 
     // Handle device list updates
-    if (this.opts.crypto && data.device_lists && data.device_lists.changed) {
-        data.device_lists.changed.forEach((u) => {
-            this.opts.crypto.userDeviceListChanged(u);
-        });
+    if (data.device_lists) {
+        if (this.opts.crypto) {
+            await this.opts.crypto.handleDeviceListChanges(
+                syncEventData, data.device_lists,
+            );
+        } else {
+            // FIXME if we *don't* have a crypto module, we still need to
+            // invalidate the device lists. But that would require a
+            // substantial bit of rework :/.
+        }
     }
 
     // Handle one_time_keys_count
@@ -1070,6 +1221,35 @@ SyncApi.prototype._pokeKeepAlive = function() {
     });
 };
 
+/**
+ * @param {Object} groupsSection Groups section object, eg. response.groups.invite
+ * @param {string} sectionName Which section this is ('invite', 'join' or 'leave')
+ */
+SyncApi.prototype._processGroupSyncEntry = function(groupsSection, sectionName) {
+    // Processes entries from 'groups' section of the sync stream
+    for (const groupId of Object.keys(groupsSection)) {
+        const groupInfo = groupsSection[groupId];
+        let group = this.client.store.getGroup(groupId);
+        const isBrandNew = group === null;
+        if (group === null) {
+            group = this.createGroup(groupId);
+        }
+        if (groupInfo.profile) {
+            group.setProfile(
+                groupInfo.profile.name, groupInfo.profile.avatar_url,
+            );
+        }
+        if (groupInfo.inviter) {
+            group.setInviter({userId: groupInfo.inviter});
+        }
+        group.setMyMembership(sectionName);
+        if (isBrandNew) {
+            // Now we've filled in all the fields, emit the Group event
+            this.client.emit("Group", group);
+        }
+    }
+};
+
 /**
  * @param {Object} obj
  * @return {Object[]}
@@ -1166,50 +1346,82 @@ SyncApi.prototype._resolveInvites = function(room) {
  */
 SyncApi.prototype._processRoomEvents = function(room, stateEventList,
                                                 timelineEventList) {
-    timelineEventList = timelineEventList || [];
-    const client = this.client;
-    // "old" and "current" state are the same initially; they
-    // start diverging if the user paginates.
-    // We must deep copy otherwise membership changes in old state
-    // will leak through to current state!
-    const oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEventList.map(function(mxEvent) {
-                return mxEvent.event;
-            }),
-        ), client.getEventMapper(),
-    );
-    const stateEvents = stateEventList;
-
-    // set the state of the room to as it was before the timeline executes
-    //
-    // XXX: what if we've already seen (some of) the events in the timeline,
-    // and they modify some of the state set in stateEvents? In that case we'll
-    // end up with the state from stateEvents, instead of the more recent state
-    // from the timeline.
-    room.oldState.setStateEvents(oldStateEvents);
-    room.currentState.setStateEvents(stateEvents);
+    // If there are no events in the timeline yet, initialise it with
+    // the given state events
+    const liveTimeline = room.getLiveTimeline();
+    const timelineWasEmpty = liveTimeline.getEvents().length == 0;
+    if (timelineWasEmpty) {
+        // Passing these events into initialiseState will freeze them, so we need
+        // to compute and cache the push actions for them now, otherwise sync dies
+        // with an attempt to assign to read only property.
+        // XXX: This is pretty horrible and is assuming all sorts of behaviour from
+        // these functions that it shouldn't be. We should probably either store the
+        // push actions cache elsewhere so we can freeze MatrixEvents, or otherwise
+        // find some solution where MatrixEvents are immutable but allow for a cache
+        // field.
+        for (const ev of stateEventList) {
+            this.client.getPushActionsForEvent(ev);
+        }
+        liveTimeline.initialiseState(stateEventList);
+    }
 
     this._resolveInvites(room);
 
     // recalculate the room name at this point as adding events to the timeline
     // may make notifications appear which should have the right name.
+    // XXX: This looks suspect: we'll end up recalculating the room once here
+    // and then again after adding events (_processSyncResponse calls it after
+    // calling us) even if no state events were added. It also means that if
+    // one of the room events in timelineEventList is something that needs
+    // a recalculation (like m.room.name) we won't recalculate until we've
+    // finished adding all the events, which will cause the notification to have
+    // the old room name rather than the new one.
     room.recalculate(this.client.credentials.userId);
 
+    // If the timeline wasn't empty, we process the state events here: they're
+    // defined as updates to the state before the start of the timeline, so this
+    // starts to roll the state forward.
+    // XXX: That's what we *should* do, but this can happen if we were previously
+    // peeking in a room, in which case we obviously do *not* want to add the
+    // state events here onto the end of the timeline. Historically, the js-sdk
+    // has just set these new state events on the old and new state. This seems
+    // very wrong because there could be events in the timeline that diverge the
+    // state, in which case this is going to leave things out of sync. However,
+    // for now I think it;s best to behave the same as the code has done previously.
+    if (!timelineWasEmpty) {
+        // XXX: As above, don't do this...
+        //room.addLiveEvents(stateEventList || []);
+        // Do this instead...
+        room.oldState.setStateEvents(stateEventList || []);
+        room.currentState.setStateEvents(stateEventList || []);
+    }
+    // execute the timeline events. This will continue to diverge the current state
+    // if the timeline has any state events in it.
+    // This also needs to be done before running push rules on the events as they need
+    // to be decorated with sender etc.
+    room.addLiveEvents(timelineEventList || []);
+};
+
+/**
+ * Takes a list of timelineEvents and adds and adds to _notifEvents
+ * as appropriate.
+ * This must be called after the room the events belong to has been stored.
+ *
+ * @param {Room} room
+ * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+ * is earlier in time. Higher index is later.
+ */
+SyncApi.prototype._processEventsForNotifs = function(room, timelineEventList) {
     // gather our notifications into this._notifEvents
-    if (client.getNotifTimelineSet()) {
+    if (this.client.getNotifTimelineSet()) {
         for (let i = 0; i < timelineEventList.length; i++) {
-            const pushActions = client.getPushActionsForEvent(timelineEventList[i]);
+            const pushActions = this.client.getPushActionsForEvent(timelineEventList[i]);
             if (pushActions && pushActions.notify &&
                 pushActions.tweaks && pushActions.tweaks.highlight) {
                 this._notifEvents.push(timelineEventList[i]);
             }
         }
     }
-
-    // execute the timeline events, this will begin to diverge the current state
-    // if the timeline has any state events in it.
-    room.addLiveEvents(timelineEventList);
 };
 
 /**
@@ -1255,7 +1467,7 @@ SyncApi.prototype._onOnline = function() {
 
 function createNewUser(client, userId) {
     const user = new User(userId);
-    reEmit(client, user, [
+    client.reEmitter.reEmit(user, [
         "User.avatarUrl", "User.displayName", "User.presence",
         "User.currentlyActive", "User.lastPresenceTs",
     ]);

src/webrtc/call.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,31 +25,6 @@ const DEBUG = true;  // set true to enable console logging.
 
 // events: hangup, error(err), replaced(call), state(state, oldState)
 
-/**
- * Fires when the MatrixCall encounters an error when sending a Matrix event.
- * <p>
- * This is required to allow errors, which occur during sending of events, to bubble up.
- * (This is because call.js does a hangup when it encounters a normal `error`, which in
- * turn could lead to an UnknownDeviceError.)
- * <p>
- * To deal with an UnknownDeviceError when trying to send events, the application should let
- * users know that there are new devices in the encrypted room (into which the event was
- * sent) and give the user the options to resend unsent events or cancel them. Resending
- * is done using {@link module:client~MatrixClient#resendEvent} and cancelling can be done by using
- * {@link module:client~MatrixClient#cancelPendingEvent}.
- * <p>
- * MatrixCall will not do anything in response to an error that causes `send_event_error`
- * to be emitted with the exception of sending `m.call.candidates`, which is retried upon
- * failure when ICE candidates are being sent. This happens during call setup.
- *
- * @event module:webrtc/call~MatrixCall#"send_event_error"
- * @param {Error} err The error caught from calling client.sendEvent in call.js.
- * @example
- * matrixCall.on("send_event_error", function(err){
- *   console.error(err);
- * });
- */
-
 /**
  * Fires whenever an error occurs when call.js encounters an issue with setting up the call.
  * <p>
@@ -110,6 +86,8 @@ function MatrixCall(opts) {
     this.mediaPromises = Object.create(null);
 
     this.screenSharingStream = null;
+
+    this._answerContent = null;
 }
 /** The length of time a call can be ringing for. */
 MatrixCall.CALL_TIMEOUT_MS = 60000;
@@ -123,6 +101,24 @@ MatrixCall.ERR_LOCAL_OFFER_FAILED = "local_offer_failed";
  */
 MatrixCall.ERR_NO_USER_MEDIA = "no_user_media";
 
+/*
+ * Error code used when a call event failed to send
+ * because unknown devices were present in the room
+ */
+MatrixCall.ERR_UNKNOWN_DEVICES = "unknown_devices";
+
+/*
+ * Error code usewd when we fail to send the invite
+ * for some reason other than there being unknown devices
+ */
+MatrixCall.ERR_SEND_INVITE = "send_invite";
+
+/*
+ * Error code usewd when we fail to send the answer
+ * for some reason other than there being unknown devices
+ */
+MatrixCall.ERR_SEND_ANSWER = "send_answer";
+
 utils.inherits(MatrixCall, EventEmitter);
 
 /**
@@ -409,6 +405,11 @@ MatrixCall.prototype.answer = function() {
     debuglog("Answering call %s of type %s", this.callId, this.type);
     const self = this;
 
+    if (self._answerContent) {
+        self._sendAnswer();
+        return;
+    }
+
     if (!this.localAVStream && !this.waitForLocalAVStream) {
         this.webRtc.getUserMedia(
             _getUserMediaVideoContraints(this.type),
@@ -457,6 +458,8 @@ MatrixCall.prototype._replacedBy = function(newCall) {
  * @param {boolean} suppressEvent True to suppress emitting an event.
  */
 MatrixCall.prototype.hangup = function(reason, suppressEvent) {
+    if (this.state == 'ended') return;
+
     debuglog("Ending call " + this.callId);
     terminate(this, "local", reason, !suppressEvent);
     const content = {
@@ -593,6 +596,28 @@ MatrixCall.prototype._maybeGotUserMediaForInvite = function(stream) {
     setState(self, 'create_offer');
 };
 
+MatrixCall.prototype._sendAnswer = function(stream) {
+    sendEvent(this, 'm.call.answer', this._answerContent).then(() => {
+        setState(this, 'connecting');
+        // If this isn't the first time we've tried to send the answer,
+        // we may have candidates queued up, so send them now.
+        _sendCandidateQueue(this);
+    }).catch((error) => {
+        // We've failed to answer: back to the ringing state
+        setState(this, 'ringing');
+        this.client.cancelPendingEvent(error.event);
+
+        let code = MatrixCall.ERR_SEND_ANSWER;
+        let message = "Failed to send answer";
+        if (error.name == 'UnknownDeviceError') {
+            code = MatrixCall.ERR_UNKNOWN_DEVICES;
+            message = "Unknown devices present in the room";
+        }
+        this.emit("error", callError(code, message));
+        throw error;
+    });
+};
+
 /**
  * Internal
  * @private
@@ -641,7 +666,7 @@ MatrixCall.prototype._maybeGotUserMediaForAnswer = function(stream) {
     self.peerConn.createAnswer(function(description) {
         debuglog("Created answer: " + description);
         self.peerConn.setLocalDescription(description, function() {
-            const content = {
+            self._answerContent = {
                 version: 0,
                 call_id: self.callId,
                 answer: {
@@ -649,8 +674,7 @@ MatrixCall.prototype._maybeGotUserMediaForAnswer = function(stream) {
                     type: self.peerConn.localDescription.type,
                 },
             };
-            sendEvent(self, 'm.call.answer', content);
-            setState(self, 'connecting');
+            self._sendAnswer();
         }, function() {
             debuglog("Error setting local description!");
         }, constraints);
@@ -671,6 +695,9 @@ MatrixCall.prototype._gotLocalIceCandidate = function(event) {
             "Got local ICE " + event.candidate.sdpMid + " candidate: " +
             event.candidate.candidate,
         );
+
+        if (this.state == 'ended') return;
+
         // As with the offer, note we need to make a copy of this object, not
         // pass the original: that broke in Chrome ~m43.
         const c = {
@@ -753,14 +780,26 @@ MatrixCall.prototype._gotLocalOffer = function(description) {
             },
             lifetime: MatrixCall.CALL_TIMEOUT_MS,
         };
-        sendEvent(self, 'm.call.invite', content);
-
-        setTimeout(function() {
-            if (self.state == 'invite_sent') {
-                self.hangup('invite_timeout');
+        sendEvent(self, 'm.call.invite', content).then(() => {
+            setState(self, 'invite_sent');
+            setTimeout(function() {
+                if (self.state == 'invite_sent') {
+                    self.hangup('invite_timeout');
+                }
+            }, MatrixCall.CALL_TIMEOUT_MS);
+        }).catch((error) => {
+            let code = MatrixCall.ERR_SEND_INVITE;
+            let message = "Failed to send invite";
+            if (error.name == 'UnknownDeviceError') {
+                code = MatrixCall.ERR_UNKNOWN_DEVICES;
+                message = "Unknown devices present in the room";
             }
-        }, MatrixCall.CALL_TIMEOUT_MS);
-        setState(self, 'invite_sent');
+
+            self.client.cancelPendingEvent(error.event);
+            terminate(self, "local", code, false);
+            self.emit("error", callError(code, message));
+            throw error;
+        });
     }, function() {
         debuglog("Error setting local description!");
     });
@@ -784,6 +823,7 @@ MatrixCall.prototype._getLocalOfferFailed = function(error) {
  * @param {Object} error
  */
 MatrixCall.prototype._getUserMediaFailed = function(error) {
+    terminate(this, "local", 'user_media_failed', false);
     this.emit(
         "error",
         callError(
@@ -792,7 +832,6 @@ MatrixCall.prototype._getUserMediaFailed = function(error) {
             "does this app have permission?",
         ),
     );
-    this.hangup("user_media_failed");
 };
 
 /**
@@ -972,17 +1011,20 @@ const setState = function(self, state) {
  * @return {Promise}
  */
 const sendEvent = function(self, eventType, content) {
-    return self.client.sendEvent(self.roomId, eventType, content).catch(
-        (err) => {
-            self.emit('send_event_error', err);
-        },
-    );
+    return self.client.sendEvent(self.roomId, eventType, content);
 };
 
 const sendCandidate = function(self, content) {
     // Sends candidates with are sent in a special way because we try to amalgamate
     // them into one message
     self.candidateSendQueue.push(content);
+
+    // Don't send the ICE candidates yet if the call is in the ringing state: this
+    // means we tried to pick (ie. started generating candidates) and then failed to
+    // send the answer and went back to the ringing state. Queue up the candidates
+    // to send if we sucessfully send the answer.
+    if (self.state == 'ringing') return;
+
     if (self.candidateSendTries === 0) {
         setTimeout(function() {
             _sendCandidateQueue(self);
@@ -1296,8 +1338,8 @@ module.exports.setVideoInput = function(deviceId) { videoInput = deviceId; };
  * Create a new Matrix call for the browser.
  * @param {MatrixClient} client The client instance to use.
  * @param {string} roomId The room the call is in.
- * @param {Object?} options optional options map.
- * @param {boolean} options.forceTURN whether relay through TURN should be forced.
+ * @param {Object?} options DEPRECATED optional options map.
+ * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be forced. This option is deprecated - use opts.forceTURN when creating the matrix client since it's only possible to set this option on outbound calls.
  * @return {MatrixCall} the call or null if the browser doesn't support calling.
  */
 module.exports.createNewMatrixCall = function(client, roomId, options) {
@@ -1350,6 +1392,9 @@ module.exports.createNewMatrixCall = function(client, roomId, options) {
             !webRtc.RtcPeerConnection || !webRtc.getUserMedia) {
         return null; // WebRTC is not supported.
     }
+
+    const optionsForceTURN = options ? options.forceTURN : false;
+
     const opts = {
         webRtc: webRtc,
         client: client,
@@ -1357,7 +1402,7 @@ module.exports.createNewMatrixCall = function(client, roomId, options) {
         roomId: roomId,
         turnServers: client.getTurnServers(),
         // call level options
-        forceTURN: options ? options.forceTURN : false,
+        forceTURN: client._forceTURN || optionsForceTURN,
     };
     return new MatrixCall(opts);
 };