v1.0.2-rc.1

Merge branch 'develop' into release-v1.0.2
Yarn needs --new-version to create a new version
2019-03-13 14:24:57 +00:00 · 2019-03-13 14:24:26 +00:00 · 2019-03-13 14:23:51 +00:00 · 2019-03-13 14:20:18 +00:00 · 2019-03-13 13:22:39 +00:00 · 2019-03-13 12:28:07 +00:00
105 changed files with 23418 additions and 3848 deletions
@@ -1,3 +1,13 @@
 {
-  "presets": ["es2015"]
+    "presets": ["es2015"],
+    "plugins": [
+        // this transforms async functions into generator functions, which
+        // are then made to use the regenerator module by babel's
+        // transform-regnerator plugin (which is enabled by es2015).
+        "transform-async-to-bluebird",
+
+        // This makes sure that the regenerator runtime is available to
+        // the transpiled code.
+        "transform-runtime",
+    ],
 }
@@ -1,7 +1,6 @@
 module.exports = {
-    parser: "babel-eslint",
+    parser: "babel-eslint", // now needed for class properties
    parserOptions: {
-        ecmaVersion: 6,
        sourceType: "module",
        ecmaFeatures: {
        }
@@ -9,6 +8,10 @@ module.exports = {
    env: {
        browser: true,
        node: true,
+
+        // babel's transform-runtime converts references to ES6 globals such as
+        // Promise and Map to core-js polyfills, so we can use ES6 globals.
+        es6: true,
    },
    extends: ["eslint:recommended", "google"],
    rules: {
@@ -63,5 +66,12 @@ module.exports = {
        "padded-blocks": ["warn"],
        "no-extend-native": ["warn"],
        "camelcase": ["warn"],
+        "no-multi-spaces": ["error", { "ignoreEOLComments": true }],
+        "space-before-function-paren": ["error", {
+            "anonymous": "never",
+            "named": "never",
+            "asyncArrow": "always",
+        }],
+        "arrow-parens": "off",
    }
 }
@@ -2,6 +2,7 @@
 /.jsdoc

 node_modules
+/.npmrc
 .lock-wscript
 build/Release
 coverage
@@ -12,6 +13,6 @@ reports
 /lib
 /specbuild

-# version file and tarball created by 'npm pack'
+# version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt
 /matrix-js-sdk-*.tgz
@@ -1,6 +1,8 @@
 language: node_js
 node_js:
-    - node # Latest stable version of nodejs.
+    - "10.11.0"
+before_install:
+    - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.13.0
+    - export PATH=$HOME/.yarn/bin:$PATH
 script:
-    - npm run lint
-    - npm run test
+    - ./travis.sh
@@ -1,3 +1,911 @@
+Changes in [1.0.2-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.2-rc.1) (2019-03-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.1...v1.0.2-rc.1)
+
+ * Use modern Yarn version on Travis CI
+   [\#858](https://github.com/matrix-org/matrix-js-sdk/pull/858)
+ * Switch to `yarn` for dependency management
+   [\#856](https://github.com/matrix-org/matrix-js-sdk/pull/856)
+ * More key request fixes
+   [\#855](https://github.com/matrix-org/matrix-js-sdk/pull/855)
+ * Calculate encrypted notification counts
+   [\#851](https://github.com/matrix-org/matrix-js-sdk/pull/851)
+ * Update dependencies
+   [\#854](https://github.com/matrix-org/matrix-js-sdk/pull/854)
+ * make sure key requests get sent
+   [\#850](https://github.com/matrix-org/matrix-js-sdk/pull/850)
+ * Use 'ideal' rather than 'exact' for deviceid
+   [\#852](https://github.com/matrix-org/matrix-js-sdk/pull/852)
+ * handle partially-shared sessions better
+   [\#848](https://github.com/matrix-org/matrix-js-sdk/pull/848)
+
+Changes in [1.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.1) (2019-03-06)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.1-rc.2...v1.0.1)
+
+ * No changes since rc.2
+
+Changes in [1.0.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.1-rc.2) (2019-03-05)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.1-rc.1...v1.0.1-rc.2)
+
+ * dont swallow txn errors in crypto store
+   [\#853](https://github.com/matrix-org/matrix-js-sdk/pull/853)
+ * Don't swallow txn errors in crypto store
+   [\#849](https://github.com/matrix-org/matrix-js-sdk/pull/849)
+
+Changes in [1.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.1-rc.1) (2019-02-28)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.0...v1.0.1-rc.1)
+
+ * Fix "e is undefined" masking the original error in MegolmDecryption
+   [\#847](https://github.com/matrix-org/matrix-js-sdk/pull/847)
+
+Changes in [1.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.0) (2019-02-14)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.0-rc.2...v1.0.0)
+
+ * Try again to commit package-lock.json
+   [\#841](https://github.com/matrix-org/matrix-js-sdk/pull/841)
+
+Changes in [1.0.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.0-rc.2) (2019-02-14)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v1.0.0-rc.1...v1.0.0-rc.2)
+
+ * Release script: commit package-lock.json
+   [\#839](https://github.com/matrix-org/matrix-js-sdk/pull/839)
+ * Add method to force re-check of key backup
+   [\#840](https://github.com/matrix-org/matrix-js-sdk/pull/840)
+ * Fix: dont check for unverified devices in left members
+   [\#838](https://github.com/matrix-org/matrix-js-sdk/pull/838)
+
+Changes in [1.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v1.0.0-rc.1) (2019-02-08)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.3...v1.0.0-rc.1)
+
+ * change hex SAS verification to decimal and emoji
+   [\#837](https://github.com/matrix-org/matrix-js-sdk/pull/837)
+ * Trust on decrypt
+   [\#836](https://github.com/matrix-org/matrix-js-sdk/pull/836)
+ * Always track our own devices
+   [\#835](https://github.com/matrix-org/matrix-js-sdk/pull/835)
+ * Make linting rules more consistent
+   [\#834](https://github.com/matrix-org/matrix-js-sdk/pull/834)
+ * add method to room to check for unverified devices
+   [\#833](https://github.com/matrix-org/matrix-js-sdk/pull/833)
+ * Merge redesign into develop
+   [\#831](https://github.com/matrix-org/matrix-js-sdk/pull/831)
+ * Supporting infrastructure for educated decisions on when to upgrade rooms
+   [\#830](https://github.com/matrix-org/matrix-js-sdk/pull/830)
+ * Include signature info for unknown devices
+   [\#826](https://github.com/matrix-org/matrix-js-sdk/pull/826)
+ * Flag v2 rooms as "safe"
+   [\#828](https://github.com/matrix-org/matrix-js-sdk/pull/828)
+ * Update ESLint
+   [\#821](https://github.com/matrix-org/matrix-js-sdk/pull/821)
+
+Changes in [0.14.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.3) (2019-01-22)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.3-rc.1...v0.14.3)
+
+ * No changes since rc.1
+
+Changes in [0.14.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.3-rc.1) (2019-01-17)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.2...v0.14.3-rc.1)
+
+ * Merge develop into experimental
+   [\#815](https://github.com/matrix-org/matrix-js-sdk/pull/815)
+ * Add a getAllEndToEndSessions to crypto store
+   [\#812](https://github.com/matrix-org/matrix-js-sdk/pull/812)
+ * T3chguy/fix displayname logic
+   [\#668](https://github.com/matrix-org/matrix-js-sdk/pull/668)
+ * Contributing: Note that rebase lets you mass signoff commits
+   [\#814](https://github.com/matrix-org/matrix-js-sdk/pull/814)
+ * take into account homoglyphs when calculating similar display names
+   [\#672](https://github.com/matrix-org/matrix-js-sdk/pull/672)
+ * Emit for key backup failures
+   [\#809](https://github.com/matrix-org/matrix-js-sdk/pull/809)
+ * emit oldEventId on "updatePendingEvent"
+   [\#646](https://github.com/matrix-org/matrix-js-sdk/pull/646)
+ * Add getThirdpartyUser to base api
+   [\#589](https://github.com/matrix-org/matrix-js-sdk/pull/589)
+ * Support custom status messages
+   [\#805](https://github.com/matrix-org/matrix-js-sdk/pull/805)
+ * Extra checks to avoid release script blowing up mid-process.
+   [\#749](https://github.com/matrix-org/matrix-js-sdk/pull/749)
+ * Move glob regex utilities out of the pushprocessor and into a more generic
+   place
+   [\#800](https://github.com/matrix-org/matrix-js-sdk/pull/800)
+
+Changes in [0.14.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.2) (2018-12-10)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.2-rc.1...v0.14.2)
+
+ * No changes since rc.1
+
+Changes in [0.14.2-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.2-rc.1) (2018-12-06)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.1...v0.14.2-rc.1)
+
+ * fix some assertions in e2e backup unit test
+   [\#794](https://github.com/matrix-org/matrix-js-sdk/pull/794)
+ * Config should be called with auth
+   [\#798](https://github.com/matrix-org/matrix-js-sdk/pull/798)
+ * Don't re-establish sessions with unknown devices
+   [\#792](https://github.com/matrix-org/matrix-js-sdk/pull/792)
+ * e2e key backups
+   [\#684](https://github.com/matrix-org/matrix-js-sdk/pull/684)
+ * WIP: online incremental megolm backups
+   [\#595](https://github.com/matrix-org/matrix-js-sdk/pull/595)
+ * Support for e2e key backups
+   [\#736](https://github.com/matrix-org/matrix-js-sdk/pull/736)
+ * Passphrase Support for e2e backups
+   [\#786](https://github.com/matrix-org/matrix-js-sdk/pull/786)
+ * Add 'getSsoLoginUrl' function
+   [\#783](https://github.com/matrix-org/matrix-js-sdk/pull/783)
+ * Fix: don't set the room name to null when heroes are missing.
+   [\#784](https://github.com/matrix-org/matrix-js-sdk/pull/784)
+ * Handle crypto db version upgrades
+   [\#785](https://github.com/matrix-org/matrix-js-sdk/pull/785)
+ * Restart broken Olm sessions
+   [\#780](https://github.com/matrix-org/matrix-js-sdk/pull/780)
+ * Use the last olm session that got a message
+   [\#776](https://github.com/matrix-org/matrix-js-sdk/pull/776)
+
+Changes in [0.14.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.1) (2018-11-22)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.0...v0.14.1)
+
+ * Warning when crypto DB is too new to use.
+
+Changes in [0.14.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.0) (2018-11-19)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.0-rc.1...v0.14.0)
+
+ * No changes since rc.1
+
+Changes in [0.14.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.0-rc.1) (2018-11-15)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.13.1...v0.14.0-rc.1)
+
+BREAKING CHANGE
+----------------
+ 
+ * js-sdk now uses Olm 3.0. Apps using Olm must update to 3.0 to
+   continue using Olm with the js-sdk. The js-sdk will call Olm's
+   init() method when the client is started.
+
+All Changes
+-----------
+
+ * Prevent messages from being sent if other messages have failed to send
+   [\#781](https://github.com/matrix-org/matrix-js-sdk/pull/781)
+ * A unit test for olm
+   [\#777](https://github.com/matrix-org/matrix-js-sdk/pull/777)
+ * Set access_token and user_id after login in with username and password.
+   [\#778](https://github.com/matrix-org/matrix-js-sdk/pull/778)
+ * Add function to get currently joined rooms.
+   [\#779](https://github.com/matrix-org/matrix-js-sdk/pull/779)
+ * Remove the request-only stuff we don't need anymore
+   [\#775](https://github.com/matrix-org/matrix-js-sdk/pull/775)
+ * Manually construct query strings for browser-request instances
+   [\#770](https://github.com/matrix-org/matrix-js-sdk/pull/770)
+ * Fix: correctly check for crypto being present
+   [\#769](https://github.com/matrix-org/matrix-js-sdk/pull/769)
+ * Update babel-eslint to 8.1.1
+   [\#768](https://github.com/matrix-org/matrix-js-sdk/pull/768)
+ * Support `request` in the browser and support supplying servers to try in
+   joinRoom()
+   [\#764](https://github.com/matrix-org/matrix-js-sdk/pull/764)
+ * loglevel should be a normal dependency
+   [\#767](https://github.com/matrix-org/matrix-js-sdk/pull/767)
+ * Stop devicelist when client is stopped
+   [\#766](https://github.com/matrix-org/matrix-js-sdk/pull/766)
+ * Update to WebAssembly-powered Olm
+   [\#743](https://github.com/matrix-org/matrix-js-sdk/pull/743)
+ * Logging lib. Fixes #332
+   [\#763](https://github.com/matrix-org/matrix-js-sdk/pull/763)
+ * Use new stop() method on matrix-mock-request
+   [\#765](https://github.com/matrix-org/matrix-js-sdk/pull/765)
+
+Changes in [0.13.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.13.1) (2018-11-14)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.13.0...v0.13.1)
+
+ * Add function to get currently joined rooms.
+   [\#779](https://github.com/matrix-org/matrix-js-sdk/pull/779)
+
+Changes in [0.13.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.13.0) (2018-11-15)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.1...v0.13.0)
+
+BREAKING CHANGE
+----------------
+ * `MatrixClient::login` now sets client `access_token` and `user_id` following successful login with username and password.
+
+Changes in [0.12.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.1) (2018-10-29)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.1-rc.1...v0.12.1)
+
+ * No changes since rc.1
+
+Changes in [0.12.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.1-rc.1) (2018-10-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.0...v0.12.1-rc.1)
+
+ * Add repository type to package.json to make it valid
+   [\#762](https://github.com/matrix-org/matrix-js-sdk/pull/762)
+ * Add getMediaConfig()
+   [\#761](https://github.com/matrix-org/matrix-js-sdk/pull/761)
+ * add new examples, to be expanded into a post
+   [\#739](https://github.com/matrix-org/matrix-js-sdk/pull/739)
+
+Changes in [0.12.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.0) (2018-10-16)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.0-rc.1...v0.12.0)
+
+ * No changes since rc.1
+
+Changes in [0.12.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.0-rc.1) (2018-10-11)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.1...v0.12.0-rc.1)
+
+BREAKING CHANGES
+----------------
+ * If js-sdk finds data in the store that is incompatible with the options currently being used,
+   it will emit sync state ERROR with an error of type InvalidStoreError. It will also stop trying
+   to sync in this situation: the app must stop the client and then either clear the store or
+   change the options (in this case, enable or disable lazy loading of members) and then start
+   the client again.
+
+All Changes
+-----------
+
+ * never replace /sync'ed memberships with OOB ones
+   [\#760](https://github.com/matrix-org/matrix-js-sdk/pull/760)
+ * Don't fail to start up if lazy load check fails
+   [\#759](https://github.com/matrix-org/matrix-js-sdk/pull/759)
+ * Make e2e work on Edge
+   [\#754](https://github.com/matrix-org/matrix-js-sdk/pull/754)
+ * throw error with same name and message over idb worker boundary
+   [\#758](https://github.com/matrix-org/matrix-js-sdk/pull/758)
+ * Default to a room version of 1 when there is no room create event
+   [\#755](https://github.com/matrix-org/matrix-js-sdk/pull/755)
+ * Silence bluebird warnings
+   [\#757](https://github.com/matrix-org/matrix-js-sdk/pull/757)
+ * allow non-ff merge from release branch into master
+   [\#750](https://github.com/matrix-org/matrix-js-sdk/pull/750)
+ * Reject with the actual error on indexeddb error
+   [\#751](https://github.com/matrix-org/matrix-js-sdk/pull/751)
+ * Update mocha to v5
+   [\#744](https://github.com/matrix-org/matrix-js-sdk/pull/744)
+ * disable lazy loading for guests as they cant create filters
+   [\#748](https://github.com/matrix-org/matrix-js-sdk/pull/748)
+ * Revert "Add getMediaLimits to client"
+   [\#745](https://github.com/matrix-org/matrix-js-sdk/pull/745)
+
+Changes in [0.11.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.1) (2018-10-01)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.1-rc.1...v0.11.1)
+
+ * No changes since rc.1
+
+Changes in [0.11.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.1-rc.1) (2018-09-27)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.0...v0.11.1-rc.1)
+
+ * make usage of hub compatible with latest version (2.5)
+   [\#747](https://github.com/matrix-org/matrix-js-sdk/pull/747)
+ * Detect when lazy loading has been toggled in client.startClient
+   [\#746](https://github.com/matrix-org/matrix-js-sdk/pull/746)
+ * Add getMediaLimits to client
+   [\#644](https://github.com/matrix-org/matrix-js-sdk/pull/644)
+ * Split npm start into an init and watch script
+   [\#742](https://github.com/matrix-org/matrix-js-sdk/pull/742)
+ * Revert "room name should only take canonical alias into account"
+   [\#738](https://github.com/matrix-org/matrix-js-sdk/pull/738)
+ * fix display name disambiguation with LL
+   [\#737](https://github.com/matrix-org/matrix-js-sdk/pull/737)
+ * Introduce Room.myMembership event
+   [\#735](https://github.com/matrix-org/matrix-js-sdk/pull/735)
+ * room name should only take canonical alias into account
+   [\#733](https://github.com/matrix-org/matrix-js-sdk/pull/733)
+ * state events from context response were not wrapped in a MatrixEvent
+   [\#732](https://github.com/matrix-org/matrix-js-sdk/pull/732)
+ * Reduce amount of promises created when inserting members
+   [\#724](https://github.com/matrix-org/matrix-js-sdk/pull/724)
+ * dont wait for LL members to be stored to resolve the members
+   [\#726](https://github.com/matrix-org/matrix-js-sdk/pull/726)
+ * RoomState.members emitted with wrong argument order for OOB members
+   [\#728](https://github.com/matrix-org/matrix-js-sdk/pull/728)
+
+Changes in [0.11.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.0) (2018-09-10)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.0-rc.1...v0.11.0)
+
+BREAKING CHANGES
+----------------
+ * v0.11.0-rc.1 introduced some breaking changes - see the respective release notes.
+
+No changes since rc.1
+
+Changes in [0.11.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.0-rc.1) (2018-09-07)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9...v0.11.0-rc.1)
+
+ * Support for lazy loading members. This should improve performance for
+   users who joined big rooms a lot. Pass to `lazyLoadMembers = true` option when calling `startClient`.
+
+BREAKING CHANGES
+----------------
+
+ * `MatrixClient::startClient` now returns a Promise. No method should be called on the client before that promise resolves. Before this method didn't return anything.
+ * A new `CATCHUP` sync state, emitted by `MatrixClient#"sync"` and returned by `MatrixClient::getSyncState()`, when doing initial sync after the `ERROR` state. See `MatrixClient` documentation for details.
+ * `RoomState::maySendEvent('m.room.message', userId)` & `RoomState::maySendMessage(userId)` do not check the membership of the user anymore, only the power level. To check if the syncing user is allowed to write in a room, use `Room::maySendMessage()` as `RoomState` is not always aware of the syncing user's membership anymore, in case lazy loading of members is enabled.
+
+All Changes
+-----------
+
+ * Only emit CATCHUP if recovering from conn error
+   [\#727](https://github.com/matrix-org/matrix-js-sdk/pull/727)
+ * Fix docstring for sync data.error
+   [\#725](https://github.com/matrix-org/matrix-js-sdk/pull/725)
+ * Re-apply "Don't rely on members to query if syncing user can post to room"
+   [\#723](https://github.com/matrix-org/matrix-js-sdk/pull/723)
+ * Revert "Don't rely on members to query if syncing user can post to room"
+   [\#721](https://github.com/matrix-org/matrix-js-sdk/pull/721)
+ * Don't rely on members to query if syncing user can post to room
+   [\#717](https://github.com/matrix-org/matrix-js-sdk/pull/717)
+ * Fixes for room.guessDMUserId
+   [\#719](https://github.com/matrix-org/matrix-js-sdk/pull/719)
+ * Fix filepanel also filtering main timeline with LL turned on.
+   [\#716](https://github.com/matrix-org/matrix-js-sdk/pull/716)
+ * Remove lazy loaded members when leaving room
+   [\#711](https://github.com/matrix-org/matrix-js-sdk/pull/711)
+ * Fix: show spinner again while recovering from connection error
+   [\#702](https://github.com/matrix-org/matrix-js-sdk/pull/702)
+ * Add method to query LL state in client
+   [\#714](https://github.com/matrix-org/matrix-js-sdk/pull/714)
+ * Fix: also load invited members when lazy loading members
+   [\#707](https://github.com/matrix-org/matrix-js-sdk/pull/707)
+ * Pass through function to discard megolm session
+   [\#704](https://github.com/matrix-org/matrix-js-sdk/pull/704)
+
+Changes in [0.10.9](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9) (2018-09-03)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9-rc.2...v0.10.9)
+
+ * No changes since rc.2
+
+Changes in [0.10.9-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9-rc.2) (2018-08-31)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9-rc.1...v0.10.9-rc.2)
+
+ * Fix for "otherMember.getAvatarUrl is not a function"
+   [\#708](https://github.com/matrix-org/matrix-js-sdk/pull/708)
+
+Changes in [0.10.9-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9-rc.1) (2018-08-30)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.8...v0.10.9-rc.1)
+
+ * Fix DM avatar
+   [\#706](https://github.com/matrix-org/matrix-js-sdk/pull/706)
+ * Lazy loading: avoid loading members at initial sync for e2e rooms
+   [\#699](https://github.com/matrix-org/matrix-js-sdk/pull/699)
+ * Improve setRoomEncryption guard against multiple m.room.encryption st…
+   [\#700](https://github.com/matrix-org/matrix-js-sdk/pull/700)
+ * Revert "Lazy loading: don't block on setting up room crypto"
+   [\#698](https://github.com/matrix-org/matrix-js-sdk/pull/698)
+ * Lazy loading: don't block on setting up room crypto
+   [\#696](https://github.com/matrix-org/matrix-js-sdk/pull/696)
+ * Add getVisibleRooms()
+   [\#695](https://github.com/matrix-org/matrix-js-sdk/pull/695)
+ * Add wrapper around getJoinedMemberCount()
+   [\#697](https://github.com/matrix-org/matrix-js-sdk/pull/697)
+ * Api to fetch events via /room/.../event/..
+   [\#694](https://github.com/matrix-org/matrix-js-sdk/pull/694)
+ * Support for room upgrades
+   [\#693](https://github.com/matrix-org/matrix-js-sdk/pull/693)
+ * Lazy loading of room members
+   [\#691](https://github.com/matrix-org/matrix-js-sdk/pull/691)
+
+Changes in [0.10.8](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.8) (2018-08-20)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.8-rc.1...v0.10.8)
+
+ * No changes since rc.1
+
+Changes in [0.10.8-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.8-rc.1) (2018-08-16)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.7...v0.10.8-rc.1)
+
+ * Add getVersion to Room
+   [\#689](https://github.com/matrix-org/matrix-js-sdk/pull/689)
+ * Add getSyncStateData()
+   [\#680](https://github.com/matrix-org/matrix-js-sdk/pull/680)
+ * Send sync error to listener
+   [\#679](https://github.com/matrix-org/matrix-js-sdk/pull/679)
+ * make sure room.tags is always a valid object to avoid crashes
+   [\#675](https://github.com/matrix-org/matrix-js-sdk/pull/675)
+ * Fix infinite spinner upon joining a room
+   [\#673](https://github.com/matrix-org/matrix-js-sdk/pull/673)
+
+Changes in [0.10.7](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.7) (2018-07-30)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.7-rc.1...v0.10.7)
+
+ * No changes since rc.1
+
+Changes in [0.10.7-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.7-rc.1) (2018-07-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.6...v0.10.7-rc.1)
+
+ * encrypt for invited users if history visibility allows.
+   [\#666](https://github.com/matrix-org/matrix-js-sdk/pull/666)
+
+Changes in [0.10.6](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.6) (2018-07-09)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.6-rc.1...v0.10.6)
+
+ * No changes since rc.1
+
+Changes in [0.10.6-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.6-rc.1) (2018-07-06)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.5...v0.10.6-rc.1)
+
+ * Expose event decryption error via Event.decrypted event
+   [\#665](https://github.com/matrix-org/matrix-js-sdk/pull/665)
+ * Add decryption error codes to base.DecryptionError
+   [\#663](https://github.com/matrix-org/matrix-js-sdk/pull/663)
+
+Changes in [0.10.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.5) (2018-06-29)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.5-rc.1...v0.10.5)
+
+ * No changes since rc.1
+
+Changes in [0.10.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.5-rc.1) (2018-06-21)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.4...v0.10.5-rc.1)
+
+ * fix auth header and filename=undefined
+   [\#659](https://github.com/matrix-org/matrix-js-sdk/pull/659)
+ * allow setting the output device for webrtc calls
+   [\#650](https://github.com/matrix-org/matrix-js-sdk/pull/650)
+ * arguments true and false are actually invalid
+   [\#596](https://github.com/matrix-org/matrix-js-sdk/pull/596)
+ * fix typo where `headers` was not being used and thus sent wrong content-type
+   [\#643](https://github.com/matrix-org/matrix-js-sdk/pull/643)
+ * fix some documentation typos
+   [\#642](https://github.com/matrix-org/matrix-js-sdk/pull/642)
+
+Changes in [0.10.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.4) (2018-06-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.4-rc.1...v0.10.4)
+
+ * No changes since rc.1
+
+Changes in [0.10.4-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.4-rc.1) (2018-06-06)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.3...v0.10.4-rc.1)
+
+ * check whether notif level is undefined, because `0` is falsey
+   [\#651](https://github.com/matrix-org/matrix-js-sdk/pull/651)
+
+Changes in [0.10.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.3) (2018-05-25)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.3-rc.1...v0.10.3)
+
+ * No changes since v0.10.3-rc.1
+
+Changes in [0.10.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.3-rc.1) (2018-05-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.2...v0.10.3-rc.1)
+
+BREAKING CHANGE
+---------------
+
+The deprecated 'callback' parameter has been removed from MatrixBaseApis.deactivateAccount
+
+ * Add `erase` option to deactivateAccount
+   [\#649](https://github.com/matrix-org/matrix-js-sdk/pull/649)
+ * Emit Session.no_consent when M_CONSENT_NOT_GIVEN received
+   [\#647](https://github.com/matrix-org/matrix-js-sdk/pull/647)
+
+Changes in [0.10.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.2) (2018-04-30)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.2-rc.1...v0.10.2)
+
+ * No changes from rc.1
+
+Changes in [0.10.2-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.2-rc.1) (2018-04-25)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.1...v0.10.2-rc.1)
+
+ * Ignore inserts of dup inbound group sessions, pt 2
+   [\#641](https://github.com/matrix-org/matrix-js-sdk/pull/641)
+ * Ignore inserts of duplicate inbound group sessions
+   [\#639](https://github.com/matrix-org/matrix-js-sdk/pull/639)
+ * Log IDB errors
+   [\#638](https://github.com/matrix-org/matrix-js-sdk/pull/638)
+ * Remove not very useful but veryv spammy log line
+   [\#632](https://github.com/matrix-org/matrix-js-sdk/pull/632)
+ * Switch event type to m.sticker.
+   [\#628](https://github.com/matrix-org/matrix-js-sdk/pull/628)
+
+Changes in [0.10.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.1) (2018-04-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0...v0.10.1)
+
+ * Log IDB errors
+   [\#638](https://github.com/matrix-org/matrix-js-sdk/pull/638)
+ * Ignore inserts of duplicate inbound group sessions
+   [\#639](https://github.com/matrix-org/matrix-js-sdk/pull/639)
+ * Ignore inserts of dup inbound group sessions, pt 2
+   [\#641](https://github.com/matrix-org/matrix-js-sdk/pull/641)
+
+Changes in [0.10.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0) (2018-04-11)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.2...v0.10.0)
+
+ * No changes
+
+Changes in [0.10.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.2) (2018-04-09)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.1...v0.10.0-rc.2)
+
+ * Add wrapper for group join API
+ * Add wrapped API to set group join\_policy
+
+Changes in [0.10.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.1) (2018-03-19)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.2...v0.10.0-rc.1)
+
+ * Fix duplicated state events in timeline from peek
+   [\#630](https://github.com/matrix-org/matrix-js-sdk/pull/630)
+ * Create indexeddb worker when starting the store
+   [\#627](https://github.com/matrix-org/matrix-js-sdk/pull/627)
+ * Fix indexeddb logging
+   [\#626](https://github.com/matrix-org/matrix-js-sdk/pull/626)
+ * Don't do /keys/changes on incremental sync
+   [\#625](https://github.com/matrix-org/matrix-js-sdk/pull/625)
+ * Don't mark devicelist dirty unnecessarily
+   [\#623](https://github.com/matrix-org/matrix-js-sdk/pull/623)
+ * Cache the joined member count for a room state
+   [\#619](https://github.com/matrix-org/matrix-js-sdk/pull/619)
+ * Fix JS doc
+   [\#618](https://github.com/matrix-org/matrix-js-sdk/pull/618)
+ * Precompute push actions for state events
+   [\#617](https://github.com/matrix-org/matrix-js-sdk/pull/617)
+ * Fix bug where global "Never send to unverified..." is ignored
+   [\#616](https://github.com/matrix-org/matrix-js-sdk/pull/616)
+ * Intern legacy top-level 'membership' field
+   [\#615](https://github.com/matrix-org/matrix-js-sdk/pull/615)
+ * Don't synthesize RR for m.room.redaction as causes the RR to go missing.
+   [\#598](https://github.com/matrix-org/matrix-js-sdk/pull/598)
+ * Make Events create Dates on demand
+   [\#613](https://github.com/matrix-org/matrix-js-sdk/pull/613)
+ * Stop cloning events when adding to state
+   [\#612](https://github.com/matrix-org/matrix-js-sdk/pull/612)
+ * De-dup code: use the initialiseState function
+   [\#611](https://github.com/matrix-org/matrix-js-sdk/pull/611)
+ * Create sentinel members on-demand
+   [\#610](https://github.com/matrix-org/matrix-js-sdk/pull/610)
+ * Some more doc on how sentinels work
+   [\#609](https://github.com/matrix-org/matrix-js-sdk/pull/609)
+ * Migrate room encryption store to crypto store
+   [\#597](https://github.com/matrix-org/matrix-js-sdk/pull/597)
+ * add parameter to getIdentityServerUrl to strip the protocol for invites
+   [\#600](https://github.com/matrix-org/matrix-js-sdk/pull/600)
+ * Move Device Tracking Data to Crypto Store
+   [\#594](https://github.com/matrix-org/matrix-js-sdk/pull/594)
+ * Optimise pushprocessor
+   [\#591](https://github.com/matrix-org/matrix-js-sdk/pull/591)
+ * Set event error before emitting
+   [\#592](https://github.com/matrix-org/matrix-js-sdk/pull/592)
+ * Add event type for stickers [WIP]
+   [\#590](https://github.com/matrix-org/matrix-js-sdk/pull/590)
+ * Migrate inbound sessions to cryptostore
+   [\#587](https://github.com/matrix-org/matrix-js-sdk/pull/587)
+ * Disambiguate names if they contain an mxid
+   [\#588](https://github.com/matrix-org/matrix-js-sdk/pull/588)
+ * Check for sessions in indexeddb before migrating
+   [\#585](https://github.com/matrix-org/matrix-js-sdk/pull/585)
+ * Emit an event for crypto store migration
+   [\#586](https://github.com/matrix-org/matrix-js-sdk/pull/586)
+ * Supporting fixes For making UnknownDeviceDialog not pop up automatically
+   [\#575](https://github.com/matrix-org/matrix-js-sdk/pull/575)
+ * Move sessions to the crypto store
+   [\#584](https://github.com/matrix-org/matrix-js-sdk/pull/584)
+ * Change crypto store transaction API
+   [\#582](https://github.com/matrix-org/matrix-js-sdk/pull/582)
+ * Add some missed copyright notices
+   [\#581](https://github.com/matrix-org/matrix-js-sdk/pull/581)
+ * Move Olm account to IndexedDB
+   [\#579](https://github.com/matrix-org/matrix-js-sdk/pull/579)
+ * Fix logging of DecryptionErrors to be more useful
+   [\#580](https://github.com/matrix-org/matrix-js-sdk/pull/580)
+ * [BREAKING] Change the behaviour of the unverfied devices blacklist flag
+   [\#568](https://github.com/matrix-org/matrix-js-sdk/pull/568)
+ * Support set_presence=offline for syncing
+   [\#557](https://github.com/matrix-org/matrix-js-sdk/pull/557)
+ * Consider cases where the sender may not redact their own event
+   [\#556](https://github.com/matrix-org/matrix-js-sdk/pull/556)
+
+Changes in [0.9.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.2) (2017-12-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.1...v0.9.2)
+
+
+Changes in [0.9.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.1) (2017-11-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0...v0.9.1)
+
+ * Fix the force TURN option
+   [\#577](https://github.com/matrix-org/matrix-js-sdk/pull/577)
+
+Changes in [0.9.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0) (2017-11-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0-rc.1...v0.9.0)
+
+
+Changes in [0.9.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0-rc.1) (2017-11-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5...v0.9.0-rc.1)
+
+ * Modify addRoomToGroup to allow setting isPublic, create alias
+   updateGroupRoomAssociation
+   [\#567](https://github.com/matrix-org/matrix-js-sdk/pull/567)
+ * Expose more functionality of pushprocessor
+   [\#565](https://github.com/matrix-org/matrix-js-sdk/pull/565)
+ * Function for working out notif trigger permission
+   [\#566](https://github.com/matrix-org/matrix-js-sdk/pull/566)
+ * keep track of event ID and timestamp of decrypted messages
+   [\#555](https://github.com/matrix-org/matrix-js-sdk/pull/555)
+ * Fix notifEvent computation
+   [\#564](https://github.com/matrix-org/matrix-js-sdk/pull/564)
+ * Fix power level of sentinel members
+   [\#563](https://github.com/matrix-org/matrix-js-sdk/pull/563)
+ * don't try to decrypt a redacted message (fixes vector-im/riot-web#3744)
+   [\#554](https://github.com/matrix-org/matrix-js-sdk/pull/554)
+ * Support room notifs
+   [\#562](https://github.com/matrix-org/matrix-js-sdk/pull/562)
+ * Fix the glob-to-regex code
+   [\#558](https://github.com/matrix-org/matrix-js-sdk/pull/558)
+
+Changes in [0.8.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5) (2017-10-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5-rc.1...v0.8.5)
+
+ * Make unknown pushrule conditions not match
+   [\#559](https://github.com/matrix-org/matrix-js-sdk/pull/559)
+
+Changes in [0.8.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5-rc.1) (2017-10-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.4...v0.8.5-rc.1)
+
+ * Implement wrapper API for removing a room from a group
+   [\#553](https://github.com/matrix-org/matrix-js-sdk/pull/553)
+ * Fix typo which resulted in stuck key download requests
+   [\#552](https://github.com/matrix-org/matrix-js-sdk/pull/552)
+ * Store group when it's created
+   [\#549](https://github.com/matrix-org/matrix-js-sdk/pull/549)
+ * Luke/groups remove rooms users from summary
+   [\#548](https://github.com/matrix-org/matrix-js-sdk/pull/548)
+ * Clean on prepublish
+   [\#546](https://github.com/matrix-org/matrix-js-sdk/pull/546)
+ * Implement wrapper APIs for adding rooms to group summary
+   [\#545](https://github.com/matrix-org/matrix-js-sdk/pull/545)
+
+Changes in [0.8.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.4) (2017-09-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3...v0.8.4)
+
+ * Fix build issue
+
+Changes in [0.8.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3) (2017-09-20)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3-rc.1...v0.8.3)
+
+ * No changes
+
+Changes in [0.8.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3-rc.1) (2017-09-19)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.2...v0.8.3-rc.1)
+
+ * consume trailing slash when creating Matrix Client in HS and IS urls
+   [\#526](https://github.com/matrix-org/matrix-js-sdk/pull/526)
+ * Add ignore users API
+   [\#539](https://github.com/matrix-org/matrix-js-sdk/pull/539)
+ * Upgrade to jsdoc 3.5.5
+   [\#540](https://github.com/matrix-org/matrix-js-sdk/pull/540)
+ * Make re-emitting events much more memory efficient
+   [\#538](https://github.com/matrix-org/matrix-js-sdk/pull/538)
+ * Only re-emit events from Event objects if needed
+   [\#536](https://github.com/matrix-org/matrix-js-sdk/pull/536)
+ * Handle 'left' users in the deviceList mananagement
+   [\#535](https://github.com/matrix-org/matrix-js-sdk/pull/535)
+ * Factor out devicelist integration tests to a separate file
+   [\#534](https://github.com/matrix-org/matrix-js-sdk/pull/534)
+ * Refactor sync._sync as an async function
+   [\#533](https://github.com/matrix-org/matrix-js-sdk/pull/533)
+ * Add es6 to eslint environments
+   [\#532](https://github.com/matrix-org/matrix-js-sdk/pull/532)
+
+Changes in [0.8.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.2) (2017-08-24)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1...v0.8.2)
+
+ * Handle m.call.* events which are decrypted asynchronously
+   [\#530](https://github.com/matrix-org/matrix-js-sdk/pull/530)
+ * Re-emit events from, er, Event objects
+   [\#529](https://github.com/matrix-org/matrix-js-sdk/pull/529)
+
+Changes in [0.8.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1) (2017-08-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.1-rc.1...v0.8.1)
+
+ * [No changes]
+
+Changes in [0.8.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.1-rc.1) (2017-08-22)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.0...v0.8.1-rc.1)
+
+ * Fix error handling in interactive-auth
+   [\#527](https://github.com/matrix-org/matrix-js-sdk/pull/527)
+ * Make lots of OlmDevice asynchronous
+   [\#524](https://github.com/matrix-org/matrix-js-sdk/pull/524)
+ * Make crypto.decryptMessage return decryption results
+   [\#523](https://github.com/matrix-org/matrix-js-sdk/pull/523)
+
+Changes in [0.8.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.0) (2017-08-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.13...v0.8.0)
+
+BREAKING CHANGE
+---------------
+
+In order to support a move to a more scalable storage backend, we need to make
+a number of the APIs related end-to-end encryption asynchronous.
+
+This release of the JS-SDK includes the following changes which will affect
+applications which support end-to-end encryption:
+
+1. `MatrixClient` now provides a new (asynchronous) method,
+   `initCrypto`. Applications which support end-to-end encryption must call
+   this method (and wait for it to complete) before calling `startClient`, to
+   give the crypto layer a chance to initialise.
+
+2. The following APIs have been changed to return promises:
+
+   * `MatrixClient.getStoredDevicesForUser`
+   * `MatrixClient.getStoredDevice`
+   * `MatrixClient.setDeviceVerified`
+   * `MatrixClient.setDeviceBlocked`
+   * `MatrixClient.setDeviceKnown`
+   * `MatrixClient.getEventSenderDeviceInfo`
+   * `MatrixClient.isEventSenderVerified`
+   * `MatrixClient.importRoomKeys`
+
+   Applications using the results of any of the above methods will need to be
+   updated to wait for the result of the promise.
+
+
+3. `MatrixClient.listDeviceKeys` has been removed altogether. It's been
+   deprecated for some time. Applications using it should instead be changed to
+   use `MatrixClient.getStoredDevices`, which is similar but returns its results
+   in a slightly different format.
+
+
+ * Make bits of `olmlib` asynchronous
+   [\#521](https://github.com/matrix-org/matrix-js-sdk/pull/521)
+ * Make some of DeviceList asynchronous
+   [\#520](https://github.com/matrix-org/matrix-js-sdk/pull/520)
+ * Make methods in crypto/algorithms async
+   [\#519](https://github.com/matrix-org/matrix-js-sdk/pull/519)
+ * Avoid sending unencrypted messages in e2e room
+   [\#518](https://github.com/matrix-org/matrix-js-sdk/pull/518)
+ * Make tests wait for syncs to happen
+   [\#517](https://github.com/matrix-org/matrix-js-sdk/pull/517)
+ * Make a load of methods in the 'Crypto' module asynchronous
+   [\#510](https://github.com/matrix-org/matrix-js-sdk/pull/510)
+ * Set `rawDisplayName` to `userId` if membership has `displayname=null`
+   [\#515](https://github.com/matrix-org/matrix-js-sdk/pull/515)
+ * Refactor handling of crypto events for async
+   [\#508](https://github.com/matrix-org/matrix-js-sdk/pull/508)
+ * Let event decryption be asynchronous
+   [\#509](https://github.com/matrix-org/matrix-js-sdk/pull/509)
+ * Transform `async` functions to bluebird promises
+   [\#511](https://github.com/matrix-org/matrix-js-sdk/pull/511)
+ * Add more group APIs
+   [\#512](https://github.com/matrix-org/matrix-js-sdk/pull/512)
+ * Retrying test: wait for localEchoUpdated event
+   [\#507](https://github.com/matrix-org/matrix-js-sdk/pull/507)
+ * Fix member events breaking on timeline reset, 2
+   [\#504](https://github.com/matrix-org/matrix-js-sdk/pull/504)
+ * Make bits of the js-sdk api asynchronous
+   [\#503](https://github.com/matrix-org/matrix-js-sdk/pull/503)
+ * Yet more js-sdk test deflakification
+   [\#499](https://github.com/matrix-org/matrix-js-sdk/pull/499)
+ * Fix racy 'matrixclient retrying' test
+   [\#497](https://github.com/matrix-org/matrix-js-sdk/pull/497)
+ * Fix spamming of key-share-requests
+   [\#495](https://github.com/matrix-org/matrix-js-sdk/pull/495)
+ * Add progress handler to `uploadContent`
+   [\#500](https://github.com/matrix-org/matrix-js-sdk/pull/500)
+ * Switch matrix-js-sdk to bluebird
+   [\#490](https://github.com/matrix-org/matrix-js-sdk/pull/490)
+ * Fix some more flakey tests
+   [\#492](https://github.com/matrix-org/matrix-js-sdk/pull/492)
+ * make the npm test script windows-friendly
+   [\#489](https://github.com/matrix-org/matrix-js-sdk/pull/489)
+ * Fix a bunch of races in the tests
+   [\#488](https://github.com/matrix-org/matrix-js-sdk/pull/488)
+ * Fix early return in MatrixClient.setGuestAccess
+   [\#487](https://github.com/matrix-org/matrix-js-sdk/pull/487)
+ * Remove testUtils.failTest
+   [\#486](https://github.com/matrix-org/matrix-js-sdk/pull/486)
+ * Add test:watch script
+   [\#485](https://github.com/matrix-org/matrix-js-sdk/pull/485)
+ * Make it possible to use async/await
+   [\#484](https://github.com/matrix-org/matrix-js-sdk/pull/484)
+ * Remove m.new_device support
+   [\#483](https://github.com/matrix-org/matrix-js-sdk/pull/483)
+ * Use access-token in header
+   [\#478](https://github.com/matrix-org/matrix-js-sdk/pull/478)
+ * Sanity-check response from /thirdparty/protocols
+   [\#482](https://github.com/matrix-org/matrix-js-sdk/pull/482)
+ * Avoid parsing plain-text errors as JSON
+   [\#479](https://github.com/matrix-org/matrix-js-sdk/pull/479)
+ * Use external mock-request
+   [\#481](https://github.com/matrix-org/matrix-js-sdk/pull/481)
+ * Fix some races in the tests
+   [\#480](https://github.com/matrix-org/matrix-js-sdk/pull/480)
+ * Fall back to MemoryCryptoStore if indexeddb fails
+   [\#475](https://github.com/matrix-org/matrix-js-sdk/pull/475)
+ * Fix load failure in firefox when indexedDB is disabled
+   [\#474](https://github.com/matrix-org/matrix-js-sdk/pull/474)
+ * Fix a race in a test
+   [\#471](https://github.com/matrix-org/matrix-js-sdk/pull/471)
+ * Avoid throwing an unhandled error when the indexeddb is deleted
+   [\#470](https://github.com/matrix-org/matrix-js-sdk/pull/470)
+ * fix jsdoc
+   [\#469](https://github.com/matrix-org/matrix-js-sdk/pull/469)
+ * Handle m.forwarded_room_key events
+   [\#468](https://github.com/matrix-org/matrix-js-sdk/pull/468)
+ * Improve error reporting from indexeddbstore.clearDatabase
+   [\#466](https://github.com/matrix-org/matrix-js-sdk/pull/466)
+ * Implement sharing of megolm keys
+   [\#454](https://github.com/matrix-org/matrix-js-sdk/pull/454)
+ * Process received room key requests
+   [\#449](https://github.com/matrix-org/matrix-js-sdk/pull/449)
+ * Send m.room_key_request events when we fail to decrypt an event
+   [\#448](https://github.com/matrix-org/matrix-js-sdk/pull/448)
+
+Changes in [0.7.13](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.13) (2017-06-22)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.12...v0.7.13)
+
+ * Fix failure on Tor browser
+   [\#473](https://github.com/matrix-org/matrix-js-sdk/pull/473)
+ * Fix issues with firefox private browsing
+   [\#472](https://github.com/matrix-org/matrix-js-sdk/pull/472)
+
+Changes in [0.7.12](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.12) (2017-06-19)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.12-rc.1...v0.7.12)
+
+ * No changes
+
+
+Changes in [0.7.12-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.12-rc.1) (2017-06-15)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.11...v0.7.12-rc.1)
+
+ * allow setting iceTransportPolicy to relay through forceTURN option
+   [\#462](https://github.com/matrix-org/matrix-js-sdk/pull/462)
+
 Changes in [0.7.11](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.11) (2017-06-12)
 ==================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.11-rc.1...v0.7.11)
@@ -113,3 +113,8 @@ include the line in your commit or pull request comment::
 can't be accepted. Git makes this trivial - just use the -s flag when you do
 ``git commit``, having first set ``user.name`` and ``user.email`` git configs
 (which you should have done anyway :)
+
+If you forgot to sign off your commits before making your pull request and are on git 2.17+ 
+you can mass signoff using rebase::
+
+    git rebase --signoff origin/develop
@@ -21,7 +21,11 @@ Please check [the working browser example](examples/browser) for more informatio
 In Node.js
 ----------

-``npm install matrix-js-sdk``
+Ensure you have the latest LTS version of Node.js installed.
+
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) if you do not have it already.
+
+``yarn add matrix-js-sdk``

 ```javascript
  var sdk = require("matrix-js-sdk");
@@ -30,9 +34,61 @@ In Node.js
    console.log("Public Rooms: %s", JSON.stringify(data));
  });
 ```
+
 See below for how to include libolm to enable end-to-end-encryption. Please check
 [the Node.js terminal app](examples/node) for a more complex example.

+To start the client:
+
+```javascript
+await client.startClient({initialSyncLimit: 10});
+```
+
+You can perform a call to `/sync` to get the current state of the client:
+
+```javascript
+client.once('sync', function(state, prevState, res) {
+    if(state === 'PREPARED') {
+        console.log("prepared");
+    } else {
+        console.log(state);
+        process.exit(1);
+    }
+});
+```
+
+To send a message:
+
+```javascript
+var content = {
+    "body": "message text",
+    "msgtype": "m.text"
+};
+client.sendEvent("roomId", "m.room.message", content, "", (err, res) => {
+    console.log(err);
+});
+```
+
+To listen for message events:
+
+```javascript
+client.on("Room.timeline", function(event, room, toStartOfTimeline) {
+  if (event.getType() !== "m.room.message") {
+    return; // only use messages
+  }
+  console.log(event.event.content.body);
+});
+```
+
+By default, the `matrix-js-sdk` client uses the `MatrixInMemoryStore` to store events as they are received. For example to iterate through the currently stored timeline for a room:
+
+```javascript
+Object.keys(client.store.rooms).forEach((roomId) => {
+  client.getRoom(roomId).timeline.forEach(t => {
+      console.log(t.event);
+  });
+});
+```

 What does this SDK do?
 ----------------------
@@ -231,7 +287,7 @@ This SDK uses JSDoc3 style comments. You can manually build and
 host the API reference from the source files like this:

 ```
-  $ npm run gendoc
+  $ yarn gendoc
  $ cd .jsdoc
  $ python -m SimpleHTTPServer 8005
 ```
@@ -245,31 +301,37 @@ The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
 [libolm](http://matrix.org/git/olm). It is left up to the application to make
 libolm available, via the ``Olm`` global.

-If the ``Olm`` global is not available, the SDK will show a warning:
+It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
+initialise the crypto layer.
+
+If the ``Olm`` global is not available, the SDK will show a warning, as shown
+below; ``initCrypto()`` will also fail.

 ```
 Unable to load crypto module: crypto will be disabled: Error: global.Olm is not defined
 ```

-The SDK will continue to work for unencrypted rooms, but it will not support
-the E2E parts of the Matrix specification.
+If the crypto layer is not (successfully) initialised, the SDK will continue to
+work for unencrypted rooms, but it will not support the E2E parts of the Matrix
+specification.

-To enable E2E support in a browser application:
+To provide the Olm library in a browser application:

 * download the transpiled libolm (from https://matrix.org/packages/npm/olm/).
 * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
 
-To enable E2E support in a node.js application:
+To provide the Olm library in a node.js application:

- * ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz``
+ * ``yarn add https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``
   (replace the URL with the latest version you want to use from
    https://matrix.org/packages/npm/olm/)
 * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.

-If you want to package Olm as dependency for your node.js application, you
-can use ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz 
--save-optional`` (if your application also works without e2e crypto enabled)
-or ``--save`` (if it doesn't) to do so.
+If you want to package Olm as dependency for your node.js application, you can
+use ``yarn add https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``. If your
+application also works without e2e crypto enabled, add ``--optional`` to mark it
+as an optional dependency.


 Contributing
@@ -279,7 +341,7 @@ want to use this SDK, skip this section.*

 First, you need to pull in the right build tools:
 ```
- $ npm install
+ $ yarn install
 ```

 Building
@@ -287,20 +349,20 @@ Building

 To build a browser version from scratch when developing::
 ```
- $ npm run build
+ $ yarn build
 ```

 To constantly do builds when files are modified (using ``watchify``)::
 ```
- $ npm run watch
+ $ yarn watch
 ```

 To run tests (Jasmine)::
 ```
- $ npm test
+ $ yarn test
 ```

 To run linting:
 ```
- $ npm run lint
+ $ yarn lint
 ```
@@ -1,13 +1,35 @@
 var matrixcs = require("./lib/matrix");
-matrixcs.request(require("browser-request"));
+const request = require('browser-request');
+const queryString = require('qs');

-matrixcs.setCryptoStoreFactory(
-    function() {
-        return new matrixcs.IndexedDBCryptoStore(
-            global.indexedDB, "matrix-js-sdk:crypto"
-        );
-    }
-);
+matrixcs.request(function(opts, fn) {
+    // We manually fix the query string for browser-request because
+    // it doesn't correctly handle cases like ?via=one&via=two. Instead
+    // we mimic `request`'s query string interface to make it all work
+    // as expected.
+    // browser-request will happily take the constructed string as the
+    // query string without trying to modify it further.
+    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
+    return request(opts, fn);
+});
+
+// just *accessing* indexedDB throws an exception in firefox with
+// indexeddb disabled.
+var indexedDB;
+try {
+    indexedDB = global.indexedDB;
+} catch(e) {}
+
+// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
+if (indexedDB) {
+    matrixcs.setCryptoStoreFactory(
+        function() {
+            return new matrixcs.IndexedDBCryptoStore(
+                indexedDB, "matrix-js-sdk:crypto"
+            );
+        }
+    );
+}

 module.exports = matrixcs; // keep export for browserify package deps
 global.matrixcs = matrixcs;
@@ -202,9 +202,9 @@ function printRoomList() {
            dateStr = new Date(msg.getTs()).toISOString().replace(
                /T/, ' ').replace(/\..+/, '');
        }
-        var me = roomList[i].getMember(myUserId);
-        if (me) {
-            fmt = fmts[me.membership];
+        var myMembership = roomList[i].getMyMembership();
+        if (myMembership) {
+            fmt = fmts[myMembership];
        }
        var roomName = fixWidth(roomList[i].name, 25);
        print(
@@ -21,4 +21,4 @@ export PATH="$rootdir/node_modules/.bin:$PATH"

 # now run our checks
 cd "$tmpdir"
-npm run lint
+yarn lint
@@ -5,8 +5,8 @@ set -x
 export NVM_DIR="$HOME/.nvm"
 [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"

-nvm use 6 || exit $?
-npm install || exit $?
+nvm use 10 || exit $?
+yarn install || exit $?

 RC=0

@@ -18,17 +18,19 @@ function fail {
 # don't use last time's test reports
 rm -rf reports coverage || exit $?

-npm test || fail "npm test finished with return code $?"
+yarn test || fail "yarn test finished with return code $?"

-npm run -s lint -- -f checkstyle > eslint.xml ||
+yarn -s lint -f checkstyle > eslint.xml ||
    fail "eslint finished with return code $?"

 # delete the old tarball, if it exists
 rm -f matrix-js-sdk-*.tgz

-npm pack ||
-    fail "npm pack finished with return code $?"
+# `yarn pack` doesn't seem to run scripts, however that seems okay here as we
+# just built as part of `install` above.
+yarn pack ||
+    fail "yarn pack finished with return code $?"

-npm run gendoc || fail "JSDoc failed with code $?"
+yarn gendoc || fail "JSDoc failed with code $?"

 exit $RC
@@ -1,22 +1,27 @@
 {
  "name": "matrix-js-sdk",
-  "version": "0.7.11",
+  "version": "1.0.2-rc.1",
  "description": "Matrix Client-Server SDK for Javascript",
  "main": "index.js",
  "scripts": {
    "test:build": "babel -s -d specbuild spec",
-    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" _mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
-    "test": "npm run test:build && npm run test:run",
-    "check": "npm run test:build && _mocha --recursive specbuild --colors",
+    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" node_modules/mocha/bin/_mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
+    "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
+    "test": "yarn test:build && yarn test:run",
+    "check": "yarn test:build && _mocha --recursive specbuild --colors",
    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
-    "start": "babel -s -w -d lib src",
+    "start": "yarn start:init && yarn start:watch",
+    "start:watch": "babel -s -w --skip-initial-build -d lib src",
+    "start:init": "babel -s -d lib src",
+    "clean": "rimraf lib dist",
    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
-    "dist": "npm run build",
+    "dist": "yarn build",
    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 113 src spec",
-    "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
+    "lint": "eslint --max-warnings 101 src spec",
+    "prepare": "yarn clean && yarn build && git rev-parse HEAD > git-revision.txt"
  },
  "repository": {
+    "type": "git",
    "url": "https://github.com/matrix-org/matrix-js-sdk"
  },
  "keywords": [
@@ -48,30 +53,41 @@
  ],
  "dependencies": {
    "another-json": "^0.2.0",
+    "babel-runtime": "^6.26.0",
+    "base-x": "3.0.4",
+    "bluebird": "^3.5.0",
    "browser-request": "^0.3.3",
-    "q": "^1.4.1",
-    "request": "^2.53.0"
+    "bs58": "^4.0.1",
+    "content-type": "^1.0.2",
+    "loglevel": "1.6.1",
+    "qs": "^6.5.2",
+    "request": "^2.88.0",
+    "unhomoglyph": "^1.0.2"
  },
  "devDependencies": {
    "babel-cli": "^6.18.0",
-    "babel-eslint": "^7.1.1",
+    "babel-eslint": "^10.0.1",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
+    "babel-plugin-transform-runtime": "^6.23.0",
    "babel-preset-es2015": "^6.18.0",
-    "browserify": "^14.0.0",
+    "browserify": "^16.2.3",
    "browserify-shim": "^3.8.13",
-    "eslint": "^3.13.1",
+    "eslint": "^5.12.0",
    "eslint-config-google": "^0.7.1",
    "exorcist": "^0.4.0",
    "expect": "^1.20.2",
    "istanbul": "^0.4.5",
-    "jsdoc": "^3.4.0",
+    "jsdoc": "^3.5.5",
    "lolex": "^1.5.2",
-    "mocha": "^3.2.0",
-    "mocha-jenkins-reporter": "^0.3.6",
+    "matrix-mock-request": "^1.2.2",
+    "mocha": "^5.2.0",
+    "mocha-jenkins-reporter": "^0.4.0",
+    "olm": "https://matrix.org/packages/npm/olm/olm-3.1.0-pre1.tgz",
    "rimraf": "^2.5.4",
    "source-map-support": "^0.4.11",
    "sourceify": "^0.1.0",
    "uglify-js": "^2.8.26",
-    "watchify": "^3.2.1"
+    "watchify": "^3.11.1"
  },
  "browserify": {
    "transform": [
@@ -6,12 +6,26 @@
 #   github-changelog-generator; install via:
 #     pip install git+https://github.com/matrix-org/github-changelog-generator.git
 #   jq; install from your distribution's package manager (https://stedolan.github.io/jq/)
-#   hub; install via brew (OSX) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
+#   hub; install via brew (macOS) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
+#   npm; typically installed by Node.js
+#   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)

 set -e

 jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
-hub --version > /dev/null || (echo "hub is required: please install it"; kill $$)
+if [[ `command -v hub` ]] && [[ `hub --version` =~ hub[[:space:]]version[[:space:]]([0-9]*).([0-9]*) ]]; then
+    HUB_VERSION_MAJOR=${BASH_REMATCH[1]}
+    HUB_VERSION_MINOR=${BASH_REMATCH[2]}
+    if [[ $HUB_VERSION_MAJOR -lt 2 ]] || [[ $HUB_VERSION_MAJOR -eq 2 && $HUB_VERSION_MINOR -lt 5 ]]; then
+        echo "hub version 2.5 is required, you have $HUB_VERSION_MAJOR.$HUB_VERSION_MINOR installed"
+        exit
+    fi
+else
+    echo "hub is required: please install it"
+    exit
+fi
+npm --version > /dev/null || (echo "npm is required: please install it"; kill $$)
+yarn --version > /dev/null || (echo "yarn is required: please install it"; kill $$)

 USAGE="$0 [-xz] [-c changelog_file] vX.Y.Z"

@@ -45,7 +59,8 @@ fi
 skip_changelog=
 skip_jsdoc=
 changelog_file="CHANGELOG.md"
-while getopts hc:xz f; do
+expected_npm_user="matrixdotorg"
+while getopts hc:u:xz f; do
    case $f in
        h)
            help
@@ -60,6 +75,9 @@ while getopts hc:xz f; do
        z)
            skip_jsdoc=1
            ;;
+        u)
+            expected_npm_user="$OPTARG"
+            ;;
    esac
 done
 shift `expr $OPTIND - 1`
@@ -74,6 +92,14 @@ if [ -z "$skip_changelog" ]; then
    update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
 fi

+# Login and publish continues to use `npm`, as it seems to have more clearly
+# defined options and semantics than `yarn` for writing to the registry.
+actual_npm_user=`npm whoami`;
+if [ $expected_npm_user != $actual_npm_user ]; then
+    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+    exit 1
+fi
+
 # ignore leading v on release
 release="${1#v}"
 tag="v${release}"
@@ -127,14 +153,22 @@ cat "${changelog_file}" | `dirname $0`/scripts/changelog_head.py > "${latest_cha
 set -x

 # Bump package.json and build the dist
-echo "npm version"
-# npm version will automatically commit its modification
+echo "yarn version"
+# yarn version will automatically commit its modification
 # and make a release tag. We don't want it to create the tag
 # because it can only sign with the default key, but we can
 # only turn off both of these behaviours, so we have to
 # manually commit the result.
-npm version --no-git-tag-version "$release"
-git commit package.json -m "$tag"
+yarn version --no-git-tag-version --new-version "$release"
+
+# commit yarn.lock if it exists, is versioned, and is modified
+if [[ -f yarn.lock && `git status --porcelain yarn.lock | grep '^ M'` ]];
+then
+    pkglock='yarn.lock'
+else
+    pkglock=''
+fi
+git commit package.json $pkglock -m "$tag"


 # figure out if we should be signing this release
@@ -150,7 +184,7 @@ fi
 # assets.
 # We make a completely separate checkout to be sure
 # we're using released versions of the dependencies
-# (rather than whatever we're pulling in from npm link)
+# (rather than whatever we're pulling in from yarn link)
 assets=''
 dodist=0
 jq -e .scripts.dist package.json 2> /dev/null || dodist=$?
@@ -161,10 +195,10 @@ if [ $dodist -eq 0 ]; then
    pushd "$builddir"
    git clone "$projdir" .
    git checkout "$rel_branch"
-    npm install
+    yarn install
    # We haven't tagged yet, so tell the dist script what version
    # it's building
-    DIST_VERSION="$tag" npm run dist
+    DIST_VERSION="$tag" yarn dist

    popd

@@ -245,7 +279,7 @@ release_text=`mktemp`
 echo "$tag" > "${release_text}"
 echo >> "${release_text}"
 cat "${latest_changes}" >> "${release_text}"
-hub release create $hubflags $assets -f "${release_text}" "$tag"
+hub release create $hubflags $assets -F "${release_text}" "$tag"

 if [ $dodist -eq 0 ]; then
    rm -rf "$builddir"
@@ -253,12 +287,13 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"

-# publish to npmjs
+# Login and publish continues to use `npm`, as it seems to have more clearly
+# defined options and semantics than `yarn` for writing to the registry.
 npm publish

 if [ -z "$skip_jsdoc" ]; then
    echo "generating jsdocs"
-    npm run gendoc
+    yarn gendoc

    echo "copying jsdocs to gh-pages branch"
    git checkout gh-pages
@@ -281,9 +316,9 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge --ff-only "$rel_branch"
+git merge "$rel_branch"

-# push master  and docs (if generated) to github
+# push master and docs (if generated) to github
 git push origin master
 if [ -z "$skip_jsdoc" ]; then
    git push origin gh-pages
@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018-2019 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,11 +18,15 @@ limitations under the License.

 "use strict";

+// load olm before the sdk if possible
+import './olm-loader';
+
 import sdk from '..';
 import testUtils from './test-utils';
-import MockHttpBackend from './mock-request';
+import MockHttpBackend from 'matrix-mock-request';
 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';
+import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';

 /**
 * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -30,21 +35,38 @@ import q from 'q';
 * @param {string} userId
 * @param {string} deviceId
 * @param {string} accessToken
+ *
+ * @param {WebStorage=} sessionStoreBackend a web storage object to use for the
+ *     session store. If undefined, we will create a MockStorageApi.
+ * @param {object} options additional options to pass to the client
 */
-export default function TestClient(userId, deviceId, accessToken) {
+export default function TestClient(
+    userId, deviceId, accessToken, sessionStoreBackend, options,
+) {
    this.userId = userId;
    this.deviceId = deviceId;

-    this.storage = new sdk.WebStorageSessionStore(new testUtils.MockStorageApi());
+    if (sessionStoreBackend === undefined) {
+        sessionStoreBackend = new testUtils.MockStorageApi();
+    }
+    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
+
    this.httpBackend = new MockHttpBackend();
-    this.client = sdk.createClient({
+
+    options = Object.assign({
        baseUrl: "http://" + userId + ".test.server",
        userId: userId,
        accessToken: accessToken,
        deviceId: deviceId,
-        sessionStore: this.storage,
+        sessionStore: sessionStore,
        request: this.httpBackend.requestFn,
-    });
+    }, options);
+    if (!options.cryptoStore) {
+        // expose this so the tests can get to it
+        this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+        options.cryptoStore = this.cryptoStore;
+    }
+    this.client = sdk.createClient(options);

    this.deviceKeys = null;
    this.oneTimeKeys = {};
@@ -74,16 +96,21 @@ TestClient.prototype.start = function() {
        pendingEventOrdering: 'detached',
    });

-    return this.httpBackend.flush().then(() => {
+    return Promise.all([
+        this.httpBackend.flushAllExpected(),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
        console.log(this + ': started');
    });
 };

 /**
 * stop the client
+ * @return {Promise} Resolves once the mock http backend has finished all pending flushes
 */
 TestClient.prototype.stop = function() {
    this.client.stopClient();
+    return this.httpBackend.stop();
 };

 /**
@@ -115,7 +142,7 @@ TestClient.prototype.expectDeviceKeyUpload = function() {
 TestClient.prototype.awaitOneTimeKeyUpload = function() {
    if (Object.keys(this.oneTimeKeys).length != 0) {
        // already got one-time keys
-        return q(this.oneTimeKeys);
+        return Promise.resolve(this.oneTimeKeys);
    }

    this.httpBackend.when("POST", "/keys/upload")
@@ -140,7 +167,8 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
              }};
          });

-    return this.httpBackend.flush('/keys/upload', 2).then((flushed) => {
+    // this can take ages
+    return this.httpBackend.flush('/keys/upload', 2, 1000).then((flushed) => {
        expect(flushed).toEqual(2);
        return this.oneTimeKeys;
    });
@@ -157,7 +185,11 @@ TestClient.prototype.expectKeyQuery = function(response) {
    this.httpBackend.when('POST', '/keys/query').respond(
        200, (path, content) => {
            Object.keys(response.device_keys).forEach((userId) => {
-                expect(content.device_keys[userId]).toEqual({});
+                expect(content.device_keys[userId]).toEqual(
+                    {},
+                    "Expected key query for " + userId + ", got " +
+                    Object.keys(content.device_keys),
+                );
            });
            return response;
        });
@@ -184,3 +216,18 @@ TestClient.prototype.getSigningKey = function() {
    const keyId = 'ed25519:' + this.deviceId;
    return this.deviceKeys.keys[keyId];
 };
+
+/**
+ * flush a single /sync request, and wait for the syncing event
+ *
+ * @returns {Promise} promise which completes once the sync has been flushed
+ */
+TestClient.prototype.flushSync = function() {
+    console.log(`${this}: flushSync`);
+    return Promise.all([
+        this.httpBackend.flush('/sync', 1),
+        testUtils.syncPromise(this.client),
+    ]).then(() => {
+        console.log(`${this}: flushSync completed`);
+    });
+};
@@ -0,0 +1,402 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import TestClient from '../TestClient';
+import testUtils from '../test-utils';
+
+const ROOM_ID = "!room:id";
+
+/**
+ * get a /sync response which contains a single e2e room (ROOM_ID), with the
+ * members given
+ *
+ * @param {string[]} roomMembers
+ *
+ * @return {object} sync response
+ */
+function getSyncResponse(roomMembers) {
+    const stateEvents = [
+        testUtils.mkEvent({
+            type: 'm.room.encryption',
+            skey: '',
+            content: {
+                algorithm: 'm.megolm.v1.aes-sha2',
+            },
+        }),
+    ];
+
+    Array.prototype.push.apply(
+        stateEvents,
+        roomMembers.map(
+            (m) => testUtils.mkMembership({
+                mship: 'join',
+                sender: m,
+            }),
+        ),
+    );
+
+    const syncResponse = {
+        next_batch: 1,
+        rooms: {
+            join: {
+                [ROOM_ID]: {
+                    state: {
+                        events: stateEvents,
+                    },
+                },
+            },
+        },
+    };
+
+    return syncResponse;
+}
+
+
+describe("DeviceList management:", function() {
+    if (!global.Olm) {
+        console.warn('not running deviceList tests: Olm not present');
+        return;
+    }
+
+    let sessionStoreBackend;
+    let aliceTestClient;
+
+    async function createTestClient() {
+        const testClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs", sessionStoreBackend,
+        );
+        await testClient.client.initCrypto();
+        return testClient;
+    }
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        // we create our own sessionStoreBackend so that we can use it for
+        // another TestClient.
+        sessionStoreBackend = new testUtils.MockStorageApi();
+
+        aliceTestClient = await createTestClient();
+    });
+
+    afterEach(function() {
+        return aliceTestClient.stop();
+    });
+
+    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+        return aliceTestClient.start().then(function() {
+            const syncResponse = getSyncResponse(['@bob:xyz']);
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+            return aliceTestClient.flushSync();
+        }).then(function() {
+            console.log("Forcing alice to download our device keys");
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {},
+                },
+            });
+
+            return Promise.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log("Telling alice to send a megolm message");
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/',
+            ).respond(200, {
+                    event_id: '$event_id',
+            });
+
+            return Promise.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
+            ]);
+        });
+    });
+
+
+    it("We should not get confused by out-of-order device query responses",
+       () => {
+           // https://github.com/vector-im/riot-web/issues/3126
+           aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
+           return aliceTestClient.start().then(() => {
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
+               return aliceTestClient.flushSync();
+           }).then(() => {
+               // to make sure the initial device queries are flushed out, we
+               // attempt to send a message.
+
+               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                   200, {
+                       device_keys: {
+                           '@bob:xyz': {},
+                           '@chris:abc': {},
+                       },
+                   },
+               );
+
+               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
+                   200, {event_id: '$event1'});
+
+               return Promise.all([
+                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
+                       () => aliceTestClient.httpBackend.flush('/send/', 1),
+                   ),
+                   aliceTestClient.client._crypto._deviceList.saveIfDirty(),
+               ]);
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   expect(data.syncToken).toEqual(1);
+               });
+
+               // invalidate bob's and chris's device lists in separate syncs
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '2',
+                   device_lists: {
+                       changed: ['@bob:xyz'],
+                   },
+               });
+               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
+                   next_batch: '3',
+                   device_lists: {
+                       changed: ['@chris:abc'],
+                   },
+               });
+               // flush both syncs
+               return aliceTestClient.flushSync().then(() => {
+                   return aliceTestClient.flushSync();
+               });
+           }).then(() => {
+               // check that we don't yet have a request for chris's devices.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@chris:abc': {},
+                   },
+                   token: '3',
+               }).respond(200, {
+                   device_keys: {'@chris:abc': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(0);
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   if (bobStat != 1 && bobStat != 2) {
+                       throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                       bobStat);
+                   }
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
+                       );
+                   }
+               });
+
+               // now add an expectation for a query for bob's devices, and let
+               // it complete.
+               aliceTestClient.httpBackend.when('POST', '/keys/query', {
+                   device_keys: {
+                       '@bob:xyz': {},
+                   },
+                   token: '2',
+               }).respond(200, {
+                   device_keys: {'@bob:xyz': {}},
+               });
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   expect(bobStat).toEqual(3);
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
+                       );
+                   }
+               });
+
+               // now let the query for chris's devices complete.
+               return aliceTestClient.httpBackend.flush('/keys/query', 1);
+           }).then((flushed) => {
+               expect(flushed).toEqual(1);
+
+               // wait for the client to stop processing the response
+               return aliceTestClient.client.downloadKeys(['@chris:abc']);
+           }).then(() => {
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   const chrisStat = data.trackingStatus['@bob:xyz'];
+
+                   expect(bobStat).toEqual(3);
+                   expect(chrisStat).toEqual(3);
+                   expect(data.syncToken).toEqual(3);
+               });
+           });
+       }).timeout(3000);
+
+    // https://github.com/vector-im/riot-web/issues/4983
+    describe("Alice should know she has stale device lists", () => {
+        beforeEach(async function() {
+            await aliceTestClient.start();
+
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, getSyncResponse(['@bob:xyz']));
+            await aliceTestClient.flushSync();
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
+                200, {
+                    device_keys: {
+                        '@bob:xyz': {},
+                    },
+                },
+            );
+            await aliceTestClient.httpBackend.flush('/keys/query', 1);
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toBeGreaterThan(
+                    0, "Alice should be tracking bob's device list",
+                );
+            });
+        });
+
+        it("when Bob leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        join: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Alice leaves", async function() {
+            aliceTestClient.httpBackend.when('GET', '/sync').respond(
+                200, {
+                    next_batch: 2,
+                    device_lists: {
+                        left: ['@bob:xyz'],
+                    },
+                    rooms: {
+                        leave: {
+                            [ROOM_ID]: {
+                                timeline: {
+                                    events: [
+                                        testUtils.mkMembership({
+                                            mship: 'leave',
+                                            sender: '@bob:xyz',
+                                        }),
+                                    ],
+                                },
+                            },
+                        },
+                    },
+                },
+            );
+
+            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();
+
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
+        });
+
+        it("when Bob leaves whilst Alice is offline", async function() {
+            aliceTestClient.stop();
+
+            const anotherTestClient = await createTestClient();
+
+            try {
+                await anotherTestClient.start();
+                anotherTestClient.httpBackend.when('GET', '/sync').respond(
+                    200, getSyncResponse([]));
+                await anotherTestClient.flushSync();
+                await anotherTestClient.client._crypto._deviceList.saveIfDirty();
+
+                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                    const bobStat = data.trackingStatus['@bob:xyz'];
+
+                    expect(bobStat).toEqual(
+                        0, "Alice should have marked bob's device list as untracked",
+                    );
+                });
+            } finally {
+                anotherTestClient.stop();
+            }
+        });
+    });
+});
@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,9 +26,13 @@ limitations under the License.

 "use strict";
 import 'source-map-support/register';
+
+// load olm before the sdk if possible
+import '../olm-loader';
+
 import expect from 'expect';
 const sdk = require("../..");
-const q = require("q");
+import Promise from 'bluebird';
 const utils = require("../../lib/utils");
 const testUtils = require("../test-utils");
 const TestClient = require('../TestClient').default;
@@ -46,7 +51,7 @@ let bobMessages;

 function bobUploadsDeviceKeys() {
    bobTestClient.expectDeviceKeyUpload();
-    return q.all([
+    return Promise.all([
        bobTestClient.client.uploadKeys(),
        bobTestClient.httpBackend.flush(),
    ]).then(() => {
@@ -67,7 +72,11 @@ function expectAliQueryKeys() {
    bobKeys[bobDeviceId] = bobTestClient.deviceKeys;
    aliTestClient.httpBackend.when("POST", "/keys/query")
            .respond(200, function(path, content) {
-        expect(content.device_keys[bobUserId]).toEqual({});
+        expect(content.device_keys[bobUserId]).toEqual(
+            {},
+            "Expected Alice to key query for " + bobUserId + ", got " +
+            Object.keys(content.device_keys),
+        );
        const result = {};
        result[bobUserId] = bobKeys;
        return {device_keys: result};
@@ -91,7 +100,11 @@ function expectBobQueryKeys() {
    bobTestClient.httpBackend.when(
        "POST", "/keys/query",
    ).respond(200, function(path, content) {
-        expect(content.device_keys[aliUserId]).toEqual({});
+        expect(content.device_keys[aliUserId]).toEqual(
+            {},
+            "Expected Bob to key query for " + aliUserId + ", got " +
+            Object.keys(content.device_keys),
+        );
        const result = {};
        result[aliUserId] = aliKeys;
        return {device_keys: result};
@@ -129,8 +142,8 @@ function expectAliClaimKeys() {
        // it can take a while to process the key query, so give it some extra
        // time, and make sure the claim actually happens rather than ploughing on
        // confusingly.
-        return aliTestClient.httpBackend.flush("/keys/claim", 1, 20).then((r) => {
-            expect(r).toEqual(1);
+        return aliTestClient.httpBackend.flush("/keys/claim", 1, 500).then((r) => {
+            expect(r).toEqual(1, "Ali did not claim Bob's keys");
        });
    });
 }
@@ -141,23 +154,24 @@ function aliDownloadsKeys() {
    expect(bobTestClient.getSigningKey()).toBeTruthy();

    const p1 = aliTestClient.client.downloadKeys([bobUserId]).then(function() {
-        expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([{
-            id: "bvcxz",
-            key: bobTestClient.getSigningKey(),
-            verified: false,
-            blocked: false,
-            display_name: null,
-        }]);
+        return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+    }).then((devices) => {
+        expect(devices.length).toEqual(1);
+        expect(devices[0].deviceId).toEqual("bvcxz");
    });
    const p2 = expectAliQueryKeys();

    // check that the localStorage is updated as we expect (not sure this is
    // an integration test, but meh)
-    return q.all([p1, p2]).then(function() {
-        const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
-        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
-        expect(devices[bobDeviceId].verified).
-            toBe(0); // DeviceVerification.UNVERIFIED
+    return Promise.all([p1, p2]).then(() => {
+        return aliTestClient.client._crypto._deviceList.saveIfDirty();
+    }).then(() => {
+        aliTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            const devices = data.devices[bobUserId];
+            expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
+            expect(devices[bobDeviceId].verified).
+                toBe(0); // DeviceVerification.UNVERIFIED
+        });
    });
 }

@@ -184,7 +198,7 @@ function bobEnablesEncryption() {
 * @return {promise} which resolves to the ciphertext for Bob's device.
 */
 function aliSendsFirstMessage() {
-    return q.all([
+    return Promise.all([
        sendMessage(aliTestClient.client),
        expectAliQueryKeys()
            .then(expectAliClaimKeys)
@@ -201,7 +215,7 @@ function aliSendsFirstMessage() {
 * @return {promise} which resolves to the ciphertext for Bob's device.
 */
 function aliSendsMessage() {
-    return q.all([
+    return Promise.all([
        sendMessage(aliTestClient.client),
        expectAliSendMessageRequest(),
    ]).spread(function(_, ciphertext) {
@@ -216,7 +230,7 @@ function aliSendsMessage() {
 * @return {promise} which resolves to the ciphertext for Ali's device.
 */
 function bobSendsReplyMessage() {
-    return q.all([
+    return Promise.all([
        sendMessage(bobTestClient.client),
        expectBobQueryKeys()
            .then(expectBobSendMessageRequest),
@@ -265,7 +279,7 @@ function sendMessage(client) {

 function expectSendMessageRequest(httpBackend) {
    const path = "/send/m.room.encrypted/";
-    const deferred = q.defer();
+    const deferred = Promise.defer();
    httpBackend.when("PUT", path).respond(200, function(path, content) {
        deferred.resolve(content);
        return {
@@ -273,8 +287,8 @@ function expectSendMessageRequest(httpBackend) {
        };
    });

-    // it can take a while to process the key query, so give it 20ms
-    return httpBackend.flush(path, 1, 20).then(() => deferred.promise);
+    // it can take a while to process the key query
+    return httpBackend.flush(path, 1).then(() => deferred.promise);
 }

 function aliRecvMessage() {
@@ -313,31 +327,37 @@ function recvMessage(httpBackend, client, sender, message) {
        },
    };
    httpBackend.when("GET", "/sync").respond(200, syncData);
-    const deferred = q.defer();
-    const onEvent = function(event) {
-        console.log(client.credentials.userId + " received event",
-                    event);

-        // ignore the m.room.member events
-        if (event.getType() == "m.room.member") {
-            return;
-        }
+    const eventPromise = new Promise((resolve, reject) => {
+        const onEvent = function(event) {
+            // ignore the m.room.member events
+            if (event.getType() == "m.room.member") {
+                return;
+            }
+            console.log(client.credentials.userId + " received event",
+                        event);

+            client.removeListener("event", onEvent);
+            resolve(event);
+        };
+        client.on("event", onEvent);
+    });
+
+    httpBackend.flush();
+
+    return eventPromise.then((event) => {
+        expect(event.isEncrypted()).toBeTruthy();
+
+        // it may still be being decrypted
+        return testUtils.awaitDecryption(event);
+    }).then((event) => {
        expect(event.getType()).toEqual("m.room.message");
        expect(event.getContent()).toEqual({
            msgtype: "m.text",
            body: "Hello, World",
        });
        expect(event.isEncrypted()).toBeTruthy();
-
-        client.removeListener("event", onEvent);
-        deferred.resolve();
-    };
-
-    client.on("event", onEvent);
-
-    httpBackend.flush();
-    return deferred.promise;
+    });
 }


@@ -375,9 +395,7 @@ function firstSync(testClient) {
    };

    testClient.httpBackend.when("GET", "/sync").respond(200, syncData);
-    return testClient.httpBackend.flush("/sync", 1).then(() => {
-        return testUtils.syncPromise(testClient.client);
-    });
+    return testClient.flushSync();
 }


@@ -386,37 +404,40 @@ describe("MatrixClient crypto", function() {
        return;
    }

-    beforeEach(function() {
+    beforeEach(async function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this

        aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
+        await aliTestClient.client.initCrypto();
+
        bobTestClient = new TestClient(bobUserId, bobDeviceId, bobAccessToken);
+        await bobTestClient.client.initCrypto();

        aliMessages = [];
        bobMessages = [];
    });

    afterEach(function() {
-        aliTestClient.stop();
        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
-        bobTestClient.stop();
        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
+
+        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
    });

    it("Bob uploads device keys", function() {
-        return q()
+        return Promise.resolve()
            .then(bobUploadsDeviceKeys);
    });

    it("Ali downloads Bobs device keys", function(done) {
-        q()
+        Promise.resolve()
            .then(bobUploadsDeviceKeys)
            .then(aliDownloadsKeys)
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
    });

    it("Ali gets keys with an invalid signature", function(done) {
-        q()
+        Promise.resolve()
            .then(bobUploadsDeviceKeys)
            .then(function() {
                // tamper bob's keys
@@ -424,14 +445,17 @@ describe("MatrixClient crypto", function() {
                expect(bobDeviceKeys.keys["curve25519:" + bobDeviceId]).toBeTruthy();
                bobDeviceKeys.keys["curve25519:" + bobDeviceId] += "abc";

-                return q.all(aliTestClient.client.downloadKeys([bobUserId]),
-                             expectAliQueryKeys());
-            })
-            .then(function() {
+                return Promise.all([
+                    aliTestClient.client.downloadKeys([bobUserId]),
+                    expectAliQueryKeys(),
+                ]);
+            }).then(function() {
+                return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+            }).then((devices) => {
                // should get an empty list
-                expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
+                expect(devices).toEqual([]);
            })
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
    });

    it("Ali gets keys with an incorrect userId", function(done) {
@@ -463,14 +487,19 @@ describe("MatrixClient crypto", function() {
            return {device_keys: result};
        });

-        q.all(
+        Promise.all([
            aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ).then(function() {
+        ]).then(function() {
+            return Promise.all([
+                aliTestClient.client.getStoredDevicesForUser(bobUserId),
+                aliTestClient.client.getStoredDevicesForUser(eveUserId),
+            ]);
+        }).spread((bobDevices, eveDevices) => {
            // should get an empty list
-            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
-            expect(aliTestClient.client.listDeviceKeys(eveUserId)).toEqual([]);
-        }).catch(testUtils.failTest).done(done);
+            expect(bobDevices).toEqual([]);
+            expect(eveDevices).toEqual([]);
+        }).nodeify(done);
    });

    it("Ali gets keys with an incorrect deviceId", function(done) {
@@ -500,18 +529,20 @@ describe("MatrixClient crypto", function() {
            return {device_keys: result};
        });

-        q.all(
+        Promise.all([
            aliTestClient.client.downloadKeys([bobUserId]),
            aliTestClient.httpBackend.flush("/keys/query", 1),
-        ).then(function() {
+        ]).then(function() {
+            return aliTestClient.client.getStoredDevicesForUser(bobUserId);
+        }).then((devices) => {
            // should get an empty list
-            expect(aliTestClient.client.listDeviceKeys(bobUserId)).toEqual([]);
-        }).catch(testUtils.failTest).done(done);
+            expect(devices).toEqual([]);
+        }).nodeify(done);
    });


    it("Bob starts his client and uploads device keys and one-time keys", function() {
-        return q()
+        return Promise.resolve()
            .then(() => bobTestClient.start())
            .then(() => bobTestClient.awaitOneTimeKeyUpload())
            .then((keys) => {
@@ -521,28 +552,30 @@ describe("MatrixClient crypto", function() {
    });

    it("Ali sends a message", function(done) {
-        q()
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .catch(testUtils.failTest).nodeify(done);
+            .nodeify(done);
    });

-    it("Bob receives a message", function(done) {
-        q()
+    it("Bob receives a message", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
            .then(aliEnablesEncryption)
            .then(aliSendsFirstMessage)
-            .then(bobRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            .then(bobRecvMessage);
    });

-    it("Bob receives a message with a bogus sender", function(done) {
-        q()
+    it("Bob receives a message with a bogus sender", function() {
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
@@ -572,34 +605,31 @@ describe("MatrixClient crypto", function() {
                };
                bobTestClient.httpBackend.when("GET", "/sync").respond(200, syncData);

-                const deferred = q.defer();
-                const onEvent = function(event) {
-                    console.log(bobUserId + " received event",
-                                event);
-
-                    // ignore the m.room.member events
-                    if (event.getType() == "m.room.member") {
-                        return;
-                    }
-
-                    expect(event.getType()).toEqual("m.room.message");
-                    expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
-                    expect(event.isEncrypted()).toBeTruthy();
-
-                    bobTestClient.client.removeListener("event", onEvent);
-                    deferred.resolve();
-                };
-
-                bobTestClient.client.on("event", onEvent);
+                const eventPromise = new Promise((resolve, reject) => {
+                    const onEvent = function(event) {
+                        console.log(bobUserId + " received event",
+                                    event);
+                        resolve(event);
+                    };
+                    bobTestClient.client.once("event", onEvent);
+                });

                bobTestClient.httpBackend.flush();
-                return deferred.promise;
-            })
-            .catch(testUtils.failTest).done(done);
+                return eventPromise;
+            }).then((event) => {
+                expect(event.isEncrypted()).toBeTruthy();
+
+                // it may still be being decrypted
+                return testUtils.awaitDecryption(event);
+            }).then((event) => {
+                expect(event.getType()).toEqual("m.room.message");
+                expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
+            });
    });

    it("Ali blocks Bob's device", function(done) {
-        q()
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
@@ -613,12 +643,13 @@ describe("MatrixClient crypto", function() {
                          // no unblocked devices, so the ciphertext should be empty
                          expect(sentContent.ciphertext).toEqual({});
                      });
-                return q.all([p1, p2]);
+                return Promise.all([p1, p2]);
            }).nodeify(done);
    });

    it("Bob receives two pre-key messages", function(done) {
-        q()
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
@@ -627,11 +658,13 @@ describe("MatrixClient crypto", function() {
            .then(bobRecvMessage)
            .then(aliSendsMessage)
            .then(bobRecvMessage)
-            .catch(testUtils.failTest).done(done);
+            .nodeify(done);
    });

    it("Bob replies to the message", function() {
-        return q()
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        bobTestClient.expectKeyQuery({device_keys: {[bobUserId]: {}}});
+        return Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => bobTestClient.start())
            .then(() => firstSync(aliTestClient))
@@ -641,68 +674,15 @@ describe("MatrixClient crypto", function() {
            .then(bobRecvMessage)
            .then(bobEnablesEncryption)
            .then(bobSendsReplyMessage).then(function(ciphertext) {
-                expect(ciphertext.type).toEqual(1);
+                expect(ciphertext.type).toEqual(1, "Unexpected cipghertext type.");
            }).then(aliRecvMessage);
    });

-
-    it("Ali does a key query when she gets a new_device event", function() {
-        return q()
-            .then(() => aliTestClient.start())
-            .then(() => firstSync(aliTestClient))
-
-            // ali will only care about bob's new_device if she is tracking
-            // bob's devices, which she will do if we enable encryption
-            .then(aliEnablesEncryption)
-
-            .then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [aliUserId]: {},
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
-            })
-
-            // make sure that the initial key download has completed
-            // (downloadKeys will wait until it does)
-            .then(() => {
-                return aliTestClient.client.downloadKeys([bobUserId]);
-            })
-
-            .then(function() {
-                const syncData = {
-                    next_batch: '2',
-                    to_device: {
-                        events: [
-                            testUtils.mkEvent({
-                                content: {
-                                    device_id: 'TEST_DEVICE',
-                                    rooms: [],
-                                },
-                                sender: bobUserId,
-                                type: 'm.new_device',
-                            }),
-                        ],
-                    },
-                };
-                aliTestClient.httpBackend.when('GET', '/sync').respond(200, syncData);
-                return aliTestClient.httpBackend.flush('/sync', 1);
-            }).then(() => {
-                aliTestClient.expectKeyQuery({
-                    device_keys: {
-                        [bobUserId]: {},
-                    },
-                });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
-            });
-    });
-
    it("Ali does a key query when encryption is enabled", function() {
        // enabling encryption in the room should make alice download devices
        // for both members.
-        return q()
+        aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
+        return Promise.resolve()
            .then(() => aliTestClient.start())
            .then(() => firstSync(aliTestClient))
            .then(() => {
@@ -732,11 +712,59 @@ describe("MatrixClient crypto", function() {
            }).then(() => {
                aliTestClient.expectKeyQuery({
                    device_keys: {
-                        [aliUserId]: {},
                        [bobUserId]: {},
                    },
                });
-                return aliTestClient.httpBackend.flush('/keys/query', 1);
+                return aliTestClient.httpBackend.flushAllExpected();
            });
    });
+
+    it("Upload new oneTimeKeys based on a /sync request - no count-asking", function() {
+        // Send a response which causes a key upload
+        const httpBackend = aliTestClient.httpBackend;
+        const syncDataEmpty = {
+            next_batch: "a",
+            device_one_time_keys_count: {
+                signed_curve25519: 0,
+            },
+        };
+
+        // enqueue expectations:
+        // * Sync with empty one_time_keys => upload keys
+
+        return Promise.resolve()
+            .then(() => {
+                console.log(aliTestClient + ': starting');
+                httpBackend.when("GET", "/pushrules").respond(200, {});
+                httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+                aliTestClient.expectDeviceKeyUpload();
+
+                // we let the client do a very basic initial sync, which it needs before
+                // it will upload one-time keys.
+                httpBackend.when("GET", "/sync").respond(200, syncDataEmpty);
+
+                aliTestClient.client.startClient({});
+
+                return httpBackend.flushAllExpected().then(() => {
+                    console.log(aliTestClient + ': started');
+                });
+            })
+            .then(() => httpBackend.when("POST", "/keys/upload")
+                .respond(200, (path, content) => {
+                    expect(content.one_time_keys).toBeTruthy();
+                    expect(content.one_time_keys).toNotEqual({});
+                    expect(Object.keys(content.one_time_keys).length)
+                        .toBeGreaterThanOrEqualTo(1);
+                    console.log('received %i one-time keys',
+                                Object.keys(content.one_time_keys).length);
+                    // cancel futher calls by telling the client
+                    // we have more than we need
+                    return {
+                       one_time_key_counts: {
+                           signed_curve25519: 70,
+                       },
+                    };
+                }))
+            .then(() => httpBackend.flushAllExpected());
+    });
 });
@@ -1,10 +1,11 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");

 import expect from 'expect';
+import Promise from 'bluebird';

 describe("MatrixClient events", function() {
    const baseUrl = "http://localhost.or.something";
@@ -29,6 +30,7 @@ describe("MatrixClient events", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("emissions", function() {
@@ -101,7 +103,7 @@ describe("MatrixClient events", function() {
        };

        it("should emit events from both the first and subsequent /sync calls",
-        function(done) {
+        function() {
            httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
            httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);

@@ -130,11 +132,16 @@ describe("MatrixClient events", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                // wait for two SYNCING events
+                utils.syncPromise(client).then(() => {
+                    return utils.syncPromise(client);
+                }),
+                httpBackend.flushAllExpected(),
+            ]).then(() => {
                expect(expectedEvents.length).toEqual(
                    0, "Failed to see all events from /sync calls",
                );
-                done();
            });
        });

@@ -157,13 +164,13 @@ describe("MatrixClient events", function() {
            });
            client.startClient();

-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                expect(fired).toBe(true, "User.presence didn't fire.");
                done();
            });
        });

-        it("should emit Room events", function(done) {
+        it("should emit Room events", function() {
            httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
            httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
            let roomInvokeCount = 0;
@@ -183,7 +190,10 @@ describe("MatrixClient events", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                expect(roomInvokeCount).toEqual(
                    1, "Room fired wrong number of times.",
                );
@@ -193,11 +203,10 @@ describe("MatrixClient events", function() {
                expect(timelineFireCount).toEqual(
                    3, "Room.timeline fired the wrong number of times",
                );
-                done();
            });
        });

-        it("should emit RoomState events", function(done) {
+        it("should emit RoomState events", function() {
            httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
            httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);

@@ -232,7 +241,10 @@ describe("MatrixClient events", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                expect(membersInvokeCount).toEqual(
                    1, "RoomState.members fired wrong number of times",
                );
@@ -242,11 +254,10 @@ describe("MatrixClient events", function() {
                expect(eventsInvokeCount).toEqual(
                    2, "RoomState.events fired wrong number of times",
                );
-                done();
            });
        });

-        it("should emit RoomMember events", function(done) {
+        it("should emit RoomMember events", function() {
            httpBackend.when("GET", "/sync").respond(200, SYNC_DATA);
            httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);

@@ -271,7 +282,10 @@ describe("MatrixClient events", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]).then(function() {
                expect(typingInvokeCount).toEqual(
                    1, "RoomMember.typing fired wrong number of times",
                );
@@ -284,11 +298,10 @@ describe("MatrixClient events", function() {
                expect(membershipInvokeCount).toEqual(
                    1, "RoomMember.membership fired wrong number of times",
                );
-                done();
            });
        });

-        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function(done) {
+        it("should emit Session.logged_out on M_UNKNOWN_TOKEN", function() {
            httpBackend.when("GET", "/sync").respond(401, { errcode: 'M_UNKNOWN_TOKEN' });

            let sessionLoggedOutCount = 0;
@@ -298,11 +311,10 @@ describe("MatrixClient events", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return httpBackend.flushAllExpected().then(function() {
                expect(sessionLoggedOutCount).toEqual(
                    1, "Session.logged_out fired wrong number of times",
                );
-                done();
            });
        });
    });
@@ -1,8 +1,8 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const EventTimeline = sdk.EventTimeline;

@@ -82,7 +82,7 @@ function startClient(httpBackend, client) {
    client.startClient();

    // set up a promise which will resolve once the client is initialised
-    const deferred = q.defer();
+    const deferred = Promise.defer();
    client.on("sync", function(state) {
        console.log("sync", state);
        if (state != "SYNCING") {
@@ -91,8 +91,10 @@ function startClient(httpBackend, client) {
        deferred.resolve();
    });

-    httpBackend.flush();
-    return deferred.promise;
+    return Promise.all([
+        httpBackend.flushAllExpected(),
+        deferred.promise,
+    ]);
 }

 describe("getEventTimeline support", function() {
@@ -109,6 +111,7 @@ describe("getEventTimeline support", function() {
        if (client) {
            client.stopClient();
        }
+        return httpBackend.stop();
    });

    it("timeline support must be enabled to work", function(done) {
@@ -125,10 +128,10 @@ describe("getEventTimeline support", function() {
            expect(function() {
                client.getEventTimeline(timelineSet, "event");
            }).toThrow();
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
    });

-    it("timeline support works when enabled", function(done) {
+    it("timeline support works when enabled", function() {
        client = sdk.createClient({
            baseUrl: baseUrl,
            userId: userId,
@@ -136,16 +139,13 @@ describe("getEventTimeline support", function() {
            timelineSupport: true,
        });

-        startClient(httpBackend, client,
-        ).then(function() {
+        return startClient(httpBackend, client).then(() => {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];
            expect(function() {
                client.getEventTimeline(timelineSet, "event");
            }).toNotThrow();
-        }).catch(utils.failTest).done(done);
-
-        httpBackend.flush().catch(utils.failTest);
+        });
    });


@@ -196,18 +196,19 @@ describe("getEventTimeline support", function() {
                },
            });

+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client, 2),
+            ]);
+        }).then(function() {
+            expect(room.timeline.length).toEqual(1);
+            expect(room.timeline[0].event).toEqual(EVENTS[1]);
+
            httpBackend.when("GET", "/messages").respond(200, {
                chunk: [EVENTS[0]],
                start: "pagin_start",
                end: "pagin_end",
            });
-
-
-            return httpBackend.flush("/sync", 2);
-        }).then(function() {
-            expect(room.timeline.length).toEqual(1);
-            expect(room.timeline[0].event).toEqual(EVENTS[1]);
-
            httpBackend.flush("/messages", 1);
            return client.scrollback(room);
        }).then(function() {
@@ -215,7 +216,7 @@ describe("getEventTimeline support", function() {
            expect(room.timeline[0].event).toEqual(EVENTS[0]);
            expect(room.timeline[1].event).toEqual(EVENTS[1]);
            expect(room.oldState.paginationToken).toEqual("pagin_end");
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
    });
 });

@@ -225,7 +226,7 @@ describe("MatrixClient event timelines", function() {
    let client = null;
    let httpBackend = null;

-    beforeEach(function(done) {
+    beforeEach(function() {
        utils.beforeEach(this); // eslint-disable-line no-invalid-this
        httpBackend = new HttpBackend();
        sdk.request(httpBackend.requestFn);
@@ -237,8 +238,7 @@ describe("MatrixClient event timelines", function() {
            timelineSupport: true,
        });

-        startClient(httpBackend, client)
-            .catch(utils.failTest).done(done);
+        return startClient(httpBackend, client);
    });

    afterEach(function() {
@@ -247,7 +247,7 @@ describe("MatrixClient event timelines", function() {
    });

    describe("getEventTimeline", function() {
-        it("should create a new timeline for new events", function(done) {
+        it("should create a new timeline for new events", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];
            httpBackend.when("GET", "/rooms/!foo%3Abar/context/event1%3Abar")
@@ -265,22 +265,23 @@ describe("MatrixClient event timelines", function() {
                    };
                });

-            client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
-                expect(tl.getEvents().length).toEqual(4);
-                for (let i = 0; i < 4; i++) {
-                    expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
-                    expect(tl.getEvents()[i].sender.name).toEqual(userName);
-                }
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1:bar").then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(4);
+                    for (let i = 0; i < 4; i++) {
+                        expect(tl.getEvents()[i].event).toEqual(EVENTS[i]);
+                        expect(tl.getEvents()[i].sender.name).toEqual(userName);
+                    }
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
        });

-        it("should return existing timeline for known events", function(done) {
+        it("should return existing timeline for known events", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];
            httpBackend.when("GET", "/sync").respond(200, {
@@ -299,17 +300,20 @@ describe("MatrixClient event timelines", function() {
                },
            });

-            httpBackend.flush("/sync").then(function() {
+            return Promise.all([
+                httpBackend.flush("/sync"),
+                utils.syncPromise(client),
+            ]).then(function() {
                return client.getEventTimeline(timelineSet, EVENTS[0].event_id);
            }).then(function(tl) {
                expect(tl.getEvents().length).toEqual(2);
                expect(tl.getEvents()[1].event).toEqual(EVENTS[0]);
                expect(tl.getEvents()[1].sender.name).toEqual(userName);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS)).toEqual("f_1_1");
-                // expect(tl.getPaginationToken(EventTimeline.FORWARDS)).toEqual("s_5_4");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                    .toEqual("f_1_1");
+                // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                //    .toEqual("s_5_4");
+            });
        });

        it("should update timelines where they overlap a previous /sync", function() {
@@ -344,7 +348,7 @@ describe("MatrixClient event timelines", function() {
                    };
                });

-            const deferred = q.defer();
+            const deferred = Promise.defer();
            client.on("sync", function() {
                client.getEventTimeline(timelineSet, EVENTS[2].event_id,
                ).then(function(tl) {
@@ -360,14 +364,14 @@ describe("MatrixClient event timelines", function() {
                        (e) => deferred.reject(e));
            });

-            return q.all([
-                httpBackend.flush(),
+            return Promise.all([
+                httpBackend.flushAllExpected(),
                deferred.promise,
            ]);
        });

        it("should join timelines where they overlap a previous /context",
-          function(done) {
+          function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];

@@ -427,41 +431,42 @@ describe("MatrixClient event timelines", function() {

            let tl0;
            let tl3;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl0 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(1);
-                tl3 = tl;
-                return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
-            }).then(function(tl) {
-                // we expect it to get merged in with event 2
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
-                expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
-                    .toBe(tl0);
-                expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
-                    .toBe(tl3);
-                expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toBe(null);
-                expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token3");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl0 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[2].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    return client.getEventTimeline(timelineSet, EVENTS[3].event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(1);
+                    tl3 = tl;
+                    return client.getEventTimeline(timelineSet, EVENTS[1].event_id);
+                }).then(function(tl) {
+                    // we expect it to get merged in with event 2
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.BACKWARDS))
+                        .toBe(tl0);
+                    expect(tl.getNeighbouringTimeline(EventTimeline.FORWARDS))
+                        .toBe(tl3);
+                    expect(tl0.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl0.getPaginationToken(EventTimeline.FORWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toBe(null);
+                    expect(tl3.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token3");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
        });

-        it("should fail gracefully if there is no event field", function(done) {
+        it("should fail gracefully if there is no event field", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];
            // we fetch event 0, then 2, then 3, and finally 1. 1 is returned
@@ -477,20 +482,21 @@ describe("MatrixClient event timelines", function() {
                    };
                });

-            client.getEventTimeline(timelineSet, "event1",
-            ).then(function(tl) {
-                // could do with a fail()
-                expect(true).toBeFalsy();
-            }).catch(function(e) {
-                expect(String(e)).toMatch(/'event'/);
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, "event1",
+                ).then(function(tl) {
+                    // could do with a fail()
+                    expect(true).toBeFalsy();
+                }, function(e) {
+                    expect(String(e)).toMatch(/'event'/);
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
        });
    });

    describe("paginateEventTimeline", function() {
-        it("should allow you to paginate backwards", function(done) {
+        it("should allow you to paginate backwards", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];

@@ -521,27 +527,28 @@ describe("MatrixClient event timelines", function() {
                });

            let tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(tl, {backwards: true});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token1");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token0");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(tl, {backwards: true});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[2]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[0]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token1");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token0");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
        });


-        it("should allow you to paginate forwards", function(done) {
+        it("should allow you to paginate forwards", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];

@@ -572,24 +579,25 @@ describe("MatrixClient event timelines", function() {
                });

            let tl;
-            client.getEventTimeline(timelineSet, EVENTS[0].event_id,
-            ).then(function(tl0) {
-                tl = tl0;
-                return client.paginateEventTimeline(
-                    tl, {backwards: false, limit: 20});
-            }).then(function(success) {
-                expect(success).toBeTruthy();
-                expect(tl.getEvents().length).toEqual(3);
-                expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
-                expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
-                expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
-                expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                    .toEqual("start_token0");
-                expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    .toEqual("end_token1");
-            }).catch(utils.failTest).done(done);
-
-            httpBackend.flush().catch(utils.failTest);
+            return Promise.all([
+                client.getEventTimeline(timelineSet, EVENTS[0].event_id,
+                ).then(function(tl0) {
+                    tl = tl0;
+                    return client.paginateEventTimeline(
+                        tl, {backwards: false, limit: 20});
+                }).then(function(success) {
+                    expect(success).toBeTruthy();
+                    expect(tl.getEvents().length).toEqual(3);
+                    expect(tl.getEvents()[0].event).toEqual(EVENTS[0]);
+                    expect(tl.getEvents()[1].event).toEqual(EVENTS[1]);
+                    expect(tl.getEvents()[2].event).toEqual(EVENTS[2]);
+                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                        .toEqual("start_token0");
+                    expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        .toEqual("end_token1");
+                }),
+                httpBackend.flushAllExpected(),
+            ]);
        });
    });

@@ -624,60 +632,66 @@ describe("MatrixClient event timelines", function() {
            });
        });

-        it("should work when /send returns before /sync", function(done) {
+        it("should work when /send returns before /sync", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];

-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                // 2 because the initial sync contained an event
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+            return Promise.all([
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    // 2 because the initial sync contained an event
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");

-                // now let the sync complete, and check it again
-                return httpBackend.flush("/sync", 1);
-            }).then(function() {
-                return utils.syncPromise(client);
-            }).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
-            }).catch(utils.failTest).done(done);
+                    // now let the sync complete, and check it again
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);
+                }),

-            httpBackend.flush("/send/m.room.message/" + TXN_ID, 1).catch(utils.failTest);
+                httpBackend.flush("/send/m.room.message/" + TXN_ID, 1),
+            ]);
        });

-        it("should work when /send returns after /sync", function(done) {
+        it("should work when /send returns after /sync", function() {
            const room = client.getRoom(roomId);
            const timelineSet = room.getTimelineSets()[0];

-            // initiate the send, and set up checks to be done when it completes
-            // - but note that it won't complete until after the /sync does, below.
-            client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
-                console.log("sendTextMessage completed");
-                expect(res.event_id).toEqual(event.event_id);
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (2)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].getContent().body).toEqual("a body");
-            }).catch(utils.failTest).done(done);
+            return Promise.all([
+                // initiate the send, and set up checks to be done when it completes
+                // - but note that it won't complete until after the /sync does, below.
+                client.sendTextMessage(roomId, "a body", TXN_ID).then(function(res) {
+                    console.log("sendTextMessage completed");
+                    expect(res.event_id).toEqual(event.event_id);
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (2)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].getContent().body).toEqual("a body");
+                }),

-            httpBackend.flush("/sync", 1).then(function() {
-                return utils.syncPromise(client);
-            }).then(function() {
-                return client.getEventTimeline(timelineSet, event.event_id);
-            }).then(function(tl) {
-                console.log("getEventTimeline completed (1)");
-                expect(tl.getEvents().length).toEqual(2);
-                expect(tl.getEvents()[1].event).toEqual(event);
+                Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
+                    return client.getEventTimeline(timelineSet, event.event_id);
+                }).then(function(tl) {
+                    console.log("getEventTimeline completed (1)");
+                    expect(tl.getEvents().length).toEqual(2);
+                    expect(tl.getEvents()[1].event).toEqual(event);

-                // now let the send complete.
-                return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
-            }).catch(utils.failTest);
+                    // now let the send complete.
+                    return httpBackend.flush("/send/m.room.message/" + TXN_ID, 1);
+                }),
+            ]);
        });
    });

@@ -714,7 +728,10 @@ describe("MatrixClient event timelines", function() {
        };
        httpBackend.when("GET", "/sync").respond(200, syncData);

-        httpBackend.flush().then(function() {
+        Promise.all([
+            httpBackend.flushAllExpected(),
+            utils.syncPromise(client),
+        ]).then(function() {
            const room = client.getRoom(roomId);
            const tl = room.getLiveTimeline();
            expect(tl.getEvents().length).toEqual(3);
@@ -739,11 +756,14 @@ describe("MatrixClient event timelines", function() {
            };
            httpBackend.when("GET", "/sync").respond(200, sync2);

-            return httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                utils.syncPromise(client),
+            ]);
        }).then(function() {
            const room = client.getRoom(roomId);
            const tl = room.getLiveTimeline();
            expect(tl.getEvents().length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
    });
 });
@@ -1,7 +1,7 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const publicGlobals = require("../../lib/matrix");
 const Room = publicGlobals.Room;
 const MatrixInMemoryStore = publicGlobals.MatrixInMemoryStore;
@@ -41,6 +41,7 @@ describe("MatrixClient", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    describe("uploadContent", function() {
@@ -49,13 +50,16 @@ describe("MatrixClient", function() {
            httpBackend.when(
                "POST", "/_matrix/media/v1/upload",
            ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                expect(req.queryParams.filename).toEqual("hi.txt");
-                expect(req.queryParams.access_token).toEqual(accessToken);
+                if (!(req.queryParams.access_token == accessToken ||
+                        req.headers["Authorization"] == "Bearer " + accessToken)) {
+                    expect(true).toBe(false);
+                }
                expect(req.headers["Content-Type"]).toEqual("text/plain");
                expect(req.opts.json).toBeFalsy();
                expect(req.opts.timeout).toBe(undefined);
-            }).respond(200, "content");
+            }).respond(200, "content", true);

            const prom = client.uploadContent({
                stream: buf,
@@ -76,7 +80,7 @@ describe("MatrixClient", function() {

                const uploads = client.getCurrentUploads();
                expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);

            httpBackend.flush();
        });
@@ -86,7 +90,7 @@ describe("MatrixClient", function() {
                "POST", "/_matrix/media/v1/upload",
            ).check(function(req) {
                expect(req.opts.json).toBeFalsy();
-            }).respond(200, JSON.stringify({ "content_uri": "uri" }));
+            }).respond(200, { "content_uri": "uri" });

            client.uploadContent({
                stream: buf,
@@ -96,22 +100,21 @@ describe("MatrixClient", function() {
                rawResponse: false,
            }).then(function(response) {
                expect(response.content_uri).toEqual("uri");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);

            httpBackend.flush();
        });

        it("should parse errors into a MatrixError", function(done) {
-            // opts.json is false, so request returns unparsed json.
            httpBackend.when(
                "POST", "/_matrix/media/v1/upload",
            ).check(function(req) {
-                expect(req.data).toEqual(buf);
+                expect(req.rawData).toEqual(buf);
                expect(req.opts.json).toBeFalsy();
-            }).respond(400, JSON.stringify({
+            }).respond(400, {
                "errcode": "M_SNAFU",
                "error": "broken",
-            }));
+            });

            client.uploadContent({
                stream: buf,
@@ -123,7 +126,7 @@ describe("MatrixClient", function() {
                expect(error.httpStatus).toEqual(400);
                expect(error.errcode).toEqual("M_SNAFU");
                expect(error.message).toEqual("broken");
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);

            httpBackend.flush();
        });
@@ -147,7 +150,7 @@ describe("MatrixClient", function() {

                const uploads = client.getCurrentUploads();
                expect(uploads.length).toEqual(0);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);

            const r = client.cancelUpload(prom);
            expect(r).toBe(true);
@@ -157,7 +160,7 @@ describe("MatrixClient", function() {
    describe("joinRoom", function() {
        it("should no-op if you've already joined a room", function() {
            const roomId = "!foo:bar";
-            const room = new Room(roomId);
+            const room = new Room(roomId, userId);
            room.addLiveEvents([
                utils.mkMembership({
                    user: userId, room: roomId, mship: "join", event: true,
@@ -303,6 +306,10 @@ describe("MatrixClient", function() {
            return;
        }

+        beforeEach(function() {
+            return client.initCrypto();
+        });
+
        it("should do an HTTP request and then store the keys", function(done) {
            const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
            // ed25519key = client.getDeviceEd25519Key();
@@ -379,7 +386,7 @@ describe("MatrixClient", function() {
                    algorithms: ["2"],
                    unsigned: { "ghi": "def" },
                });
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);

            httpBackend.flush();
        });
@@ -396,7 +403,7 @@ describe("MatrixClient", function() {

            client.deleteDevice(
                "my_device", auth,
-            ).catch(utils.failTest).done(done);
+            ).nodeify(done);

            httpBackend.flush();
        });
@@ -2,10 +2,11 @@
 import 'source-map-support/register';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");

 import expect from 'expect';
+import Promise from 'bluebird';

 describe("MatrixClient opts", function() {
    const baseUrl = "http://localhost.or.something";
@@ -63,6 +64,7 @@ describe("MatrixClient opts", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    describe("without opts.store", function() {
@@ -93,7 +95,7 @@ describe("MatrixClient opts", function() {
            httpBackend.flush("/txn1", 1);
        });

-        it("should be able to sync / get new events", function(done) {
+        it("should be able to sync / get new events", async function() {
            const expectedEventTypes = [ // from /initialSync
                "m.room.message", "m.room.name", "m.room.member", "m.room.member",
                "m.room.create",
@@ -109,19 +111,16 @@ describe("MatrixClient opts", function() {
            httpBackend.when("GET", "/pushrules").respond(200, {});
            httpBackend.when("POST", "/filter").respond(200, { filter_id: "foo" });
            httpBackend.when("GET", "/sync").respond(200, syncData);
-            client.startClient();
-            httpBackend.flush("/pushrules", 1).then(function() {
-                return httpBackend.flush("/filter", 1);
-            }).then(function() {
-                return httpBackend.flush("/sync", 1);
-            }).then(function() {
-                return utils.syncPromise(client);
-            }).done(function() {
-                expect(expectedEventTypes.length).toEqual(
-                    0, "Expected to see event types: " + expectedEventTypes,
-                );
-                done();
-            });
+            await client.startClient();
+            await httpBackend.flush("/pushrules", 1);
+            await httpBackend.flush("/filter", 1);
+            await Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]);
+            expect(expectedEventTypes.length).toEqual(
+                0, "Expected to see event types: " + expectedEventTypes,
+            );
        });
    });

@@ -1,7 +1,9 @@
 "use strict";
 import 'source-map-support/register';
+import Promise from 'bluebird';
+
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const EventStatus = sdk.EventStatus;

@@ -34,6 +36,7 @@ describe("MatrixClient retrying", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    xit("should retry according to MatrixScheduler.retryFn", function() {
@@ -52,15 +55,20 @@ describe("MatrixClient retrying", function() {

    });

-    it("should mark events as EventStatus.CANCELLED when cancelled", function(done) {
+    it("should mark events as EventStatus.CANCELLED when cancelled", function() {
        // send a couple of events; the second will be queued
-        client.sendMessage(roomId, "m1").then(function(ev) {
-            expect(ev).toEqual(ev1);
-        });
-        client.sendMessage(roomId, "m2").then(function(ev) {
-            expect(ev).toEqual(ev2);
+        const p1 = client.sendMessage(roomId, "m1").then(function(ev) {
+            // we expect the first message to fail
+            throw new Error('Message 1 unexpectedly sent successfully');
+        }, (e) => {
+            // this is expected
        });

+        // XXX: it turns out that the promise returned by this message
+        // never gets resolved.
+        // https://github.com/matrix-org/matrix-js-sdk/issues/496
+        client.sendMessage(roomId, "m2");
+
        // both events should be in the timeline at this point
        const tl = room.getLiveTimeline().getEvents();
        expect(tl.length).toEqual(2);
@@ -86,7 +94,14 @@ describe("MatrixClient retrying", function() {
            }).toThrow();
        }).respond(400); // fail the first message

-        httpBackend.flush().then(function() {
+        // wait for the localecho of ev1 to be updated
+        const p3 = new Promise((resolve, reject) => {
+            room.on("Room.localEchoUpdated", (ev0) => {
+                if(ev0 === ev1) {
+                    resolve();
+                }
+            });
+        }).then(function() {
            expect(ev1.status).toEqual(EventStatus.NOT_SENT);
            expect(tl.length).toEqual(1);

@@ -94,7 +109,13 @@ describe("MatrixClient retrying", function() {
            client.cancelPendingEvent(ev1);
            expect(ev1.status).toEqual(EventStatus.CANCELLED);
            expect(tl.length).toEqual(0);
-        }).catch(utils.failTest).done(done);
+        });
+
+        return Promise.all([
+            p1,
+            p3,
+            httpBackend.flushAllExpected(),
+        ]);
    });

    describe("resending", function() {
@@ -2,9 +2,10 @@
 import 'source-map-support/register';
 const sdk = require("../..");
 const EventStatus = sdk.EventStatus;
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");

+import Promise from 'bluebird';
 import expect from 'expect';

 describe("MatrixClient room timelines", function() {
@@ -129,6 +130,7 @@ describe("MatrixClient room timelines", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("local echo events", function() {
@@ -251,11 +253,14 @@ describe("MatrixClient room timelines", function() {
                client.scrollback(room).done(function() {
                    expect(room.timeline.length).toEqual(1);
                    expect(room.oldState.paginationToken).toBe(null);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
            });
            httpBackend.flush("/sync", 1);
        });
@@ -317,11 +322,14 @@ describe("MatrixClient room timelines", function() {
                    expect(oldMsg.sender.name).toEqual("Old Alice");
                    const newMsg = room.timeline[3];
                    expect(newMsg.sender.name).toEqual(userName);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
            });
            httpBackend.flush("/sync", 1);
        });
@@ -348,11 +356,14 @@ describe("MatrixClient room timelines", function() {
                    expect(room.timeline.length).toEqual(3);
                    expect(room.timeline[0].event).toEqual(sbEvents[1]);
                    expect(room.timeline[1].event).toEqual(sbEvents[0]);
-                    done();
+
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1);
            });
            httpBackend.flush("/sync", 1);
        });
@@ -376,9 +387,11 @@ describe("MatrixClient room timelines", function() {
                    expect(room.oldState.paginationToken).toEqual(sbEndTok);
                });

-                httpBackend.flush("/sync", 1);
                httpBackend.flush("/messages", 1).done(function() {
-                    done();
+                    // still have a sync to flush
+                    httpBackend.flush("/sync", 1).then(() => {
+                        done();
+                    });
                });
            });
            httpBackend.flush("/sync", 1);
@@ -386,17 +399,17 @@ describe("MatrixClient room timelines", function() {
    });

    describe("new events", function() {
-        it("should be added to the right place in the timeline", function(done) {
+        it("should be added to the right place in the timeline", function() {
            const eventData = [
                utils.mkMessage({user: userId, room: roomId}),
                utils.mkMessage({user: userId, room: roomId}),
            ];
            setNextSyncData(eventData);

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);

                let index = 0;
@@ -408,9 +421,10 @@ describe("MatrixClient room timelines", function() {
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    expect(index).toEqual(2);
                    expect(room.timeline.length).toEqual(3);
                    expect(room.timeline[2].event).toEqual(
@@ -419,12 +433,11 @@ describe("MatrixClient room timelines", function() {
                    expect(room.timeline[1].event).toEqual(
                        eventData[0],
                    );
-                }).catch(utils.failTest).done(done);
+                });
            });
-            httpBackend.flush("/sync", 1);
        });

-        it("should set the right event.sender values", function(done) {
+        it("should set the right event.sender values", function() {
            const eventData = [
                utils.mkMessage({user: userId, room: roomId}),
                utils.mkMembership({
@@ -435,24 +448,24 @@ describe("MatrixClient room timelines", function() {
            eventData[1].__prev_event = USER_MEMBERSHIP_EVENT;
            setNextSyncData(eventData);

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    const preNameEvent = room.timeline[room.timeline.length - 3];
                    const postNameEvent = room.timeline[room.timeline.length - 1];
                    expect(preNameEvent.sender.name).toEqual(userName);
                    expect(postNameEvent.sender.name).toEqual("New Name");
-                }).catch(utils.failTest).done(done);
+                });
            });
-            httpBackend.flush("/sync", 1);
        });

-        it("should set the right room.name", function(done) {
+        it("should set the right room.name", function() {
            const secondRoomNameEvent = utils.mkEvent({
                user: userId, room: roomId, type: "m.room.name", content: {
                    name: "Room 2",
@@ -461,19 +474,20 @@ describe("MatrixClient room timelines", function() {
            secondRoomNameEvent.__prev_event = ROOM_NAME_EVENT;
            setNextSyncData([secondRoomNameEvent]);

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);
                let nameEmitCount = 0;
                client.on("Room.name", function(rm) {
                    nameEmitCount += 1;
                });

-                httpBackend.flush("/sync", 1).then(() => {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    expect(nameEmitCount).toEqual(1);
                    expect(room.name).toEqual("Room 2");
                    // do another round
@@ -485,19 +499,18 @@ describe("MatrixClient room timelines", function() {
                    thirdRoomNameEvent.__prev_event = secondRoomNameEvent;
                    setNextSyncData([thirdRoomNameEvent]);
                    httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);
-                    httpBackend.flush("/sync", 1).then(() => {
-                        return utils.syncPromise(client);
-                    }).done(function() {
-                        expect(nameEmitCount).toEqual(2);
-                        expect(room.name).toEqual("Room 3");
-                        done();
-                    });
+                    return Promise.all([
+                        httpBackend.flush("/sync", 1),
+                        utils.syncPromise(client),
+                    ]);
+                }).then(function() {
+                    expect(nameEmitCount).toEqual(2);
+                    expect(room.name).toEqual("Room 3");
                });
            });
-            httpBackend.flush("/sync", 1);
        });

-        it("should set the right room members", function(done) {
+        it("should set the right room members", function() {
            const userC = "@cee:bar";
            const userD = "@dee:bar";
            const eventData = [
@@ -512,14 +525,15 @@ describe("MatrixClient room timelines", function() {
            eventData[1].__prev_event = null;
            setNextSyncData(eventData);

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).then(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    expect(room.currentState.getMembers().length).toEqual(4);
                    expect(room.currentState.getMember(userC).name).toEqual("C");
                    expect(room.currentState.getMember(userC).membership).toEqual(
@@ -529,30 +543,30 @@ describe("MatrixClient room timelines", function() {
                    expect(room.currentState.getMember(userD).membership).toEqual(
                        "invite",
                    );
-                }).catch(utils.failTest).done(done);
+                });
            });
-            httpBackend.flush("/sync", 1);
        });
    });

    describe("gappy sync", function() {
-        it("should copy the last known state to the new timeline", function(done) {
+        it("should copy the last known state to the new timeline", function() {
            const eventData = [
                utils.mkMessage({user: userId, room: roomId}),
            ];
            setNextSyncData(eventData);
            NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    expect(room.timeline.length).toEqual(1);
                    expect(room.timeline[0].event).toEqual(eventData[0]);
                    expect(room.currentState.getMembers().length).toEqual(2);
@@ -564,23 +578,21 @@ describe("MatrixClient room timelines", function() {
                    expect(room.currentState.getMember(otherUserId).membership).toEqual(
                        "join",
                    );
-                    done();
                });
            });
-            httpBackend.flush("/sync", 1);
        });

-        it("should emit a 'Room.timelineReset' event", function(done) {
+        it("should emit a 'Room.timelineReset' event", function() {
            const eventData = [
                utils.mkMessage({user: userId, room: roomId}),
            ];
            setNextSyncData(eventData);
            NEXT_SYNC_DATA.rooms.join[roomId].timeline.limited = true;

-            client.on("sync", function(state) {
-                if (state !== "PREPARED") {
-                    return;
-                }
+            return Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]).then(() => {
                const room = client.getRoom(roomId);

                let emitCount = 0;
@@ -590,14 +602,13 @@ describe("MatrixClient room timelines", function() {
                });

                httpBackend.flush("/messages", 1);
-                httpBackend.flush("/sync", 1).then(function() {
-                    return utils.syncPromise(client);
-                }).done(function() {
+                return Promise.all([
+                    httpBackend.flush("/sync", 1),
+                    utils.syncPromise(client),
+                ]).then(function() {
                    expect(emitCount).toEqual(1);
-                    done();
                });
            });
-            httpBackend.flush("/sync", 1);
        });
    });
 });
@@ -1,12 +1,13 @@
 "use strict";
 import 'source-map-support/register';
 const sdk = require("../..");
-const HttpBackend = require("../mock-request");
+const HttpBackend = require("matrix-mock-request");
 const utils = require("../test-utils");
 const MatrixEvent = sdk.MatrixEvent;
 const EventTimeline = sdk.EventTimeline;

 import expect from 'expect';
+import Promise from 'bluebird';

 describe("MatrixClient syncing", function() {
    const baseUrl = "http://localhost.or.something";
@@ -37,6 +38,7 @@ describe("MatrixClient syncing", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("startClient", function() {
@@ -51,7 +53,7 @@ describe("MatrixClient syncing", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                done();
            });
        });
@@ -65,7 +67,7 @@ describe("MatrixClient syncing", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            httpBackend.flushAllExpected().done(function() {
                done();
            });
        });
@@ -113,7 +115,7 @@ describe("MatrixClient syncing", function() {
            };
        });

-        it("should resolve incoming invites from /sync", function(done) {
+        it("should resolve incoming invites from /sync", function() {
            syncData.rooms.join[roomOne].state.events.push(
                utils.mkMembership({
                    room: roomOne, mship: "invite", user: userC,
@@ -132,17 +134,20 @@ describe("MatrixClient syncing", function() {
                resolveInvitesToProfiles: true,
            });

-            httpBackend.flush().done(function() {
+
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const member = client.getRoom(roomOne).getMember(userC);
                expect(member.name).toEqual("The Boss");
                expect(
                    member.getAvatarUrl("home.server.url", null, null, null, false),
                ).toBeTruthy();
-                done();
            });
        });

-        it("should use cached values from m.presence wherever possible", function(done) {
+        it("should use cached values from m.presence wherever possible", function() {
            syncData.presence.events = [
                utils.mkPresence({
                    user: userC, presence: "online", name: "The Ghost",
@@ -160,14 +165,16 @@ describe("MatrixClient syncing", function() {
                resolveInvitesToProfiles: true,
            });

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const member = client.getRoom(roomOne).getMember(userC);
                expect(member.name).toEqual("The Ghost");
-                done();
            });
        });

-        it("should result in events on the room member firing", function(done) {
+        it("should result in events on the room member firing", function() {
            syncData.presence.events = [
                utils.mkPresence({
                    user: userC, presence: "online", name: "The Ghost",
@@ -192,13 +199,15 @@ describe("MatrixClient syncing", function() {
                resolveInvitesToProfiles: true,
            });

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                expect(latestFiredName).toEqual("The Ghost");
-                done();
            });
        });

-        it("should no-op if resolveInvitesToProfiles is not set", function(done) {
+        it("should no-op if resolveInvitesToProfiles is not set", function() {
            syncData.rooms.join[roomOne].state.events.push(
                utils.mkMembership({
                    room: roomOne, mship: "invite", user: userC,
@@ -209,13 +218,15 @@ describe("MatrixClient syncing", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const member = client.getRoom(roomOne).getMember(userC);
                expect(member.name).toEqual(userC);
                expect(
                    member.getAvatarUrl("home.server.url", null, null, null, false),
                ).toBe(null);
-                done();
            });
        });
    });
@@ -236,15 +247,17 @@ describe("MatrixClient syncing", function() {
        };

        it("should create users for presence events from /sync",
-        function(done) {
+        function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                expect(client.getUser(userA).presence).toEqual("online");
                expect(client.getUser(userB).presence).toEqual("unavailable");
-                done();
            });
        });
    });
@@ -355,57 +368,66 @@ describe("MatrixClient syncing", function() {
            },
        };

-        it("should continually recalculate the right room name.", function(done) {
+        it("should continually recalculate the right room name.", function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);
            httpBackend.when("GET", "/sync").respond(200, nextSyncData);

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                const room = client.getRoom(roomOne);
                // should have clobbered the name to the one from /events
                expect(room.name).toEqual(
                    nextSyncData.rooms.join[roomOne].state.events[0].content.name,
                );
-                done();
            });
        });

-        it("should store the right events in the timeline.", function(done) {
+        it("should store the right events in the timeline.", function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);
            httpBackend.when("GET", "/sync").respond(200, nextSyncData);

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                const room = client.getRoom(roomTwo);
                // should have added the message from /events
                expect(room.timeline.length).toEqual(2);
                expect(room.timeline[1].getContent().body).toEqual(msgText);
-                done();
            });
        });

-        it("should set the right room name.", function(done) {
+        it("should set the right room name.", function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);
            httpBackend.when("GET", "/sync").respond(200, nextSyncData);

            client.startClient();
-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                const room = client.getRoom(roomTwo);
                // should use the display name of the other person.
                expect(room.name).toEqual(otherDisplayName);
-                done();
            });
        });

-        it("should set the right user's typing flag.", function(done) {
+        it("should set the right user's typing flag.", function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);
            httpBackend.when("GET", "/sync").respond(200, nextSyncData);

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
                const room = client.getRoom(roomTwo);
                let member = room.getMember(otherUserId);
                expect(member).toBeTruthy();
@@ -413,7 +435,34 @@ describe("MatrixClient syncing", function() {
                member = room.getMember(selfUserId);
                expect(member).toBeTruthy();
                expect(member.typing).toEqual(false);
-                done();
+            });
+        });
+
+        // XXX: This test asserts that the js-sdk obeys the spec and treats state
+        // events that arrive in the incremental sync as if they preceeded the
+        // timeline events, however this breaks peeking, so it's disabled
+        // (see sync.js)
+        xit("should correctly interpret state in incremental sync.", function() {
+            httpBackend.when("GET", "/sync").respond(200, syncData);
+            httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+            client.startClient();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
+                const stateAtStart = room.getLiveTimeline().getState(
+                    EventTimeline.BACKWARDS,
+                );
+                const startRoomNameEvent = stateAtStart.getStateEvents('m.room.name', '');
+                expect(startRoomNameEvent.getContent().name).toEqual('Old room name');
+
+                const stateAtEnd = room.getLiveTimeline().getState(
+                    EventTimeline.FORWARDS,
+                );
+                const endRoomNameEvent = stateAtEnd.getStateEvents('m.room.name', '');
+                expect(endRoomNameEvent.getContent().name).toEqual('A new room name');
            });
        });

@@ -448,10 +497,13 @@ describe("MatrixClient syncing", function() {
            httpBackend.when("GET", "/sync").respond(200, syncData);

            client.startClient();
-            httpBackend.flush();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]);
        });

-        it("should set the back-pagination token on new rooms", function(done) {
+        it("should set the back-pagination token on new rooms", function() {
            const syncData = {
                next_batch: "batch_token",
                rooms: {
@@ -471,16 +523,19 @@ describe("MatrixClient syncing", function() {

            httpBackend.when("GET", "/sync").respond(200, syncData);

-            httpBackend.flush().then(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const room = client.getRoom(roomTwo);
+                expect(room).toExist();
                const tok = room.getLiveTimeline()
                    .getPaginationToken(EventTimeline.BACKWARDS);
                expect(tok).toEqual("roomtwotok");
-                done();
-            }).catch(utils.failTest).done();
+            });
        });

-        it("should set the back-pagination token on gappy syncs", function(done) {
+        it("should set the back-pagination token on gappy syncs", function() {
            const syncData = {
                next_batch: "batch_token",
                rooms: {
@@ -511,13 +566,15 @@ describe("MatrixClient syncing", function() {
                expect(tok).toEqual("newerTok");
            });

-            httpBackend.flush().then(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const room = client.getRoom(roomOne);
                const tl = room.getLiveTimeline();
                expect(tl.getEvents().length).toEqual(1);
                expect(resetCallCount).toEqual(1);
-                done();
-            }).catch(utils.failTest).done();
+            });
        });
    });

@@ -570,7 +627,7 @@ describe("MatrixClient syncing", function() {
            };
        });

-        it("should sync receipts from /sync.", function(done) {
+        it("should sync receipts from /sync.", function() {
            const ackEvent = syncData.rooms.join[roomOne].timeline.events[0];
            const receipt = {};
            receipt[ackEvent.event_id] = {
@@ -588,7 +645,10 @@ describe("MatrixClient syncing", function() {

            client.startClient();

-            httpBackend.flush().done(function() {
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(),
+            ]).then(function() {
                const room = client.getRoom(roomOne);
                expect(room.getReceiptsForEvent(new MatrixEvent(ackEvent))).toEqual([{
                    type: "m.read",
@@ -597,7 +657,6 @@ describe("MatrixClient syncing", function() {
                        ts: 176592842636,
                    },
                }]);
-                done();
            });
        });
    });
@@ -617,7 +676,7 @@ describe("MatrixClient syncing", function() {
        beforeEach(function(done) {
            client.startClient();

-            httpBackend.flush().then(function() {
+            httpBackend.flushAllExpected().then(function() {
                // the /sync call from syncLeftRooms ends up in the request
                // queue behind the call from the running client; add a response
                // to flush the client's one out.
@@ -627,29 +686,34 @@ describe("MatrixClient syncing", function() {
            });
        });

-        it("should create and use an appropriate filter", function(done) {
+        it("should create and use an appropriate filter", function() {
            httpBackend.when("POST", "/filter").check(function(req) {
                expect(req.data).toEqual({
                    room: { timeline: {limit: 1},
                            include_leave: true }});
            }).respond(200, { filter_id: "another_id" });

+            const defer = Promise.defer();
+
            httpBackend.when("GET", "/sync").check(function(req) {
                expect(req.queryParams.filter).toEqual("another_id");
-                done();
+                defer.resolve();
            }).respond(200, {});

            client.syncLeftRooms();

            // first flush the filter request; this will make syncLeftRooms
            // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                // flush the syncs
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+            return Promise.all([
+                httpBackend.flush("/filter").then(function() {
+                    // flush the syncs
+                    return httpBackend.flushAllExpected();
+                }),
+                defer.promise,
+            ]);
        });

-        it("should set the back-pagination token on left rooms", function(done) {
+        it("should set the back-pagination token on left rooms", function() {
            const syncData = {
                next_batch: "batch_token",
                rooms: {
@@ -674,20 +738,31 @@ describe("MatrixClient syncing", function() {

            httpBackend.when("GET", "/sync").respond(200, syncData);

-            client.syncLeftRooms().then(function() {
-                const room = client.getRoom(roomTwo);
-                const tok = room.getLiveTimeline().getPaginationToken(
-                    EventTimeline.BACKWARDS);
+            return Promise.all([
+                client.syncLeftRooms().then(function() {
+                    const room = client.getRoom(roomTwo);
+                    const tok = room.getLiveTimeline().getPaginationToken(
+                        EventTimeline.BACKWARDS);

-                expect(tok).toEqual("pagTok");
-                done();
-            }).catch(utils.failTest).done();
+                    expect(tok).toEqual("pagTok");
+                }),

-            // first flush the filter request; this will make syncLeftRooms
-            // make its /sync call
-            httpBackend.flush("/filter").then(function() {
-                return httpBackend.flush();
-            }).catch(utils.failTest);
+                // first flush the filter request; this will make syncLeftRooms
+                // make its /sync call
+                httpBackend.flush("/filter").then(function() {
+                    return httpBackend.flushAllExpected();
+                }),
+            ]);
        });
    });
+
+    /**
+     * waits for the MatrixClient to emit one or more 'sync' events.
+     *
+     * @param {Number?} numSyncs number of syncs to wait for
+     * @returns {Promise} promise which resolves after the sync events have happened
+     */
+    function awaitSyncEvent(numSyncs) {
+        return utils.syncPromise(client, numSyncs);
+    }
 });
@@ -16,18 +16,10 @@ limitations under the License.

 "use strict";

-import 'source-map-support/register';
-
-let Olm = null;
-try {
-    Olm = require('olm');
-} catch (e) {}
-
 const anotherjson = require('another-json');
-const q = require('q');
+import Promise from 'bluebird';
 import expect from 'expect';

-const sdk = require('../..');
 const utils = require('../../lib/utils');
 const testUtils = require('../test-utils');
 const TestClient = require('../TestClient').default;
@@ -46,7 +38,7 @@ function createOlmSession(olmAccount, recipientTestClient) {
        const otkId = utils.keys(keys)[0];
        const otk = keys[otkId];

-        const session = new Olm.Session();
+        const session = new global.Olm.Session();
        session.create_outbound(
            olmAccount, recipientTestClient.getDeviceKey(), otk.key,
        );
@@ -127,6 +119,7 @@ function encryptMegolmEvent(opts) {
    }

    return {
+        event_id: 'test_megolm_event',
        content: {
            algorithm: "m.megolm.v1.aes-sha2",
            ciphertext: opts.groupSession.encrypt(JSON.stringify(plaintext)),
@@ -209,9 +202,11 @@ function getSyncResponse(roomMembers) {


 describe("megolm", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!global.Olm) {
+        console.warn('not running megolm tests: Olm not present');
        return;
    }
+    const Olm = global.Olm;

    let testOlmAccount;
    let testSenderKey;
@@ -286,12 +281,13 @@ describe("megolm", function() {
        return claimResponse;
    }

-    beforeEach(function() {
+    beforeEach(async function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this

        aliceTestClient = new TestClient(
            "@alice:localhost", "xzcvb", "akjgkrgjs",
        );
+        await aliceTestClient.client.initCrypto();

        testOlmAccount = new Olm.Account();
        testOlmAccount.create();
@@ -300,10 +296,10 @@ describe("megolm", function() {
    });

    afterEach(function() {
-        aliceTestClient.stop();
+        return aliceTestClient.stop();
    });

-    it("Alice receives a megolm message", function(done) {
+    it("Alice receives a megolm message", function() {
        return aliceTestClient.start().then(() => {
            return createOlmSession(testOlmAccount, aliceTestClient);
        }).then((p2pSession) => {
@@ -343,17 +339,18 @@ describe("megolm", function() {
            };

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
+            expect(event.isEncrypted()).toBe(true);
+            return testUtils.awaitDecryption(event);
+        }).then((event) => {
            expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
    });

-    it("Alice receives a megolm message before the session keys", function(done) {
+    it("Alice receives a megolm message before the session keys", function() {
        // https://github.com/vector-im/riot-web/issues/2273
        let roomKeyEncrypted;

@@ -393,9 +390,7 @@ describe("megolm", function() {
            };

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -410,17 +405,27 @@ describe("megolm", function() {
            };

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
+
+            if (event.getContent().msgtype != 'm.bad.encrypted') {
+                return event;
+            }
+
+            return new Promise((resolve, reject) => {
+                event.once('Event.decrypted', (ev) => {
+                    console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+                    resolve(ev);
+                });
+            });
+        }).then((event) => {
            expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
    });

-    it("Alice gets a second room_key message", function(done) {
+    it("Alice gets a second room_key message", function() {
        return aliceTestClient.start().then(() => {
            return createOlmSession(testOlmAccount, aliceTestClient);
        }).then((p2pSession) => {
@@ -480,19 +485,21 @@ describe("megolm", function() {
            };
            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse2);

-            return aliceTestClient.httpBackend.flush("/sync", 2);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            // flush both syncs
+            return aliceTestClient.flushSync().then(() => {
+                return aliceTestClient.flushSync();
+            });
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
            expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
    });

-    it('Alice sends a megolm message', function(done) {
+    it('Alice sends a megolm message', function() {
        let p2pSession;

+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
            return createOlmSession(testOlmAccount, aliceTestClient);
@@ -510,22 +517,20 @@ describe("megolm", function() {
            syncResponse.to_device = { events: [olmEvent] };

            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            // start out with the device unknown - the send should be rejected.
            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
                200, getTestKeysQueryResponse('@bob:xyz'),
            );

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
                    throw new Error("sendTextMessage failed on an unknown device");
                }, (e) => {
                    expect(e.name).toEqual("UnknownDeviceError");
                }),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
            ]);
        }).then(function() {
            // mark the device as known, and resend.
@@ -565,52 +570,19 @@ describe("megolm", function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const pendingMsg = room.getPendingEvents()[0];

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.resendEvent(pendingMsg, room),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
            ]);
-        }).nodeify(done);
+        });
    });

-    it("Alice shouldn't do a second /query for non-e2e-capable devices", function(done) {
-        return aliceTestClient.start().then(function() {
-            const syncResponse = getSyncResponse(['@bob:xyz']);
-            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-
-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
-        }).then(function() {
-            console.log("Forcing alice to download our device keys");
-
-            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
-                device_keys: {
-                    '@bob:xyz': {},
-                },
-            });
-
-            return q.all([
-                aliceTestClient.client.downloadKeys(['@bob:xyz']),
-                aliceTestClient.httpBackend.flush('/keys/query', 1),
-            ]);
-        }).then(function() {
-            console.log("Telling alice to send a megolm message");
-
-            aliceTestClient.httpBackend.when(
-                'PUT', '/send/',
-            ).respond(200, {
-                    event_id: '$event_id',
-            });
-
-            return q.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
-            ]);
-        }).nodeify(done);
-    });
-
-
-    it("We shouldn't attempt to send to blocked devices", function(done) {
+    it("We shouldn't attempt to send to blocked devices", function() {
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
            return createOlmSession(testOlmAccount, aliceTestClient);
@@ -626,9 +598,7 @@ describe("megolm", function() {
            syncResponse.to_device = { events: [olmEvent] };
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            console.log('Forcing alice to download our device keys');

@@ -636,7 +606,7 @@ describe("megolm", function() {
                200, getTestKeysQueryResponse('@bob:xyz'),
            );

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.downloadKeys(['@bob:xyz']),
                aliceTestClient.httpBackend.flush('/keys/query', 1),
            ]);
@@ -651,17 +621,22 @@ describe("megolm", function() {
                event_id: '$event_id',
            });

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
            ]);
-        }).nodeify(done);
+        });
    });

-    it("We should start a new megolm session when a device is blocked", function(done) {
+    it("We should start a new megolm session when a device is blocked", function() {
        let p2pSession;
        let megolmSessionId;

+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
            return createOlmSession(testOlmAccount, aliceTestClient);
@@ -679,9 +654,7 @@ describe("megolm", function() {
            syncResponse.to_device = { events: [olmEvent] };
            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);

-            return aliceTestClient.httpBackend.flush('/sync', 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            console.log("Fetching bob's devices and marking known");

@@ -689,9 +662,9 @@ describe("megolm", function() {
                200, getTestKeysQueryResponse('@bob:xyz'),
            );

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.downloadKeys(['@bob:xyz']),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
            ]).then((keys) => {
                aliceTestClient.client.setDeviceKnown('@bob:xyz', 'DEVICE_ID');
            });
@@ -722,9 +695,13 @@ describe("megolm", function() {
                };
            });

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
            ]);
        }).then(function() {
            console.log('Telling alice to block our device');
@@ -741,15 +718,15 @@ describe("megolm", function() {
                };
            });

-            return q.all([
+            return Promise.all([
                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test2'),
-                aliceTestClient.httpBackend.flush(),
+                aliceTestClient.httpBackend.flushAllExpected(),
            ]);
-        }).nodeify(done);
+        });
    });

    // https://github.com/vector-im/riot-web/issues/2676
-    it("Alice should send to her other devices", function(done) {
+    it("Alice should send to her other devices", function() {
        // for this test, we make the testOlmAccount be another of Alice's devices.
        // it ought to get included in messages Alice sends.

@@ -791,20 +768,17 @@ describe("megolm", function() {
                getTestKeysQueryResponse(aliceTestClient.userId),
            );

-            return aliceTestClient.httpBackend.flush();
+            return aliceTestClient.httpBackend.flushAllExpected();
        }).then(function() {
            // start out with the device unknown - the send should be rejected.
-            return q.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
-                    throw new Error("sendTextMessage failed on an unknown device");
-                }, (e) => {
-                    expect(e.name).toEqual("UnknownDeviceError");
-                    expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
-                    expect(Object.keys(e.devices[aliceTestClient.userId])).
-                        toEqual(['DEVICE_ID']);
-                }),
-                aliceTestClient.httpBackend.flush(),
-            ]);
+            return aliceTestClient.client.sendTextMessage(ROOM_ID, 'test').then(() => {
+                throw new Error("sendTextMessage failed on an unknown device");
+            }, (e) => {
+                expect(e.name).toEqual("UnknownDeviceError");
+                expect(Object.keys(e.devices)).toEqual([aliceTestClient.userId]);
+                expect(Object.keys(e.devices[aliceTestClient.userId])).
+                    toEqual(['DEVICE_ID']);
+            });
        }).then(function() {
            // mark the device as known, and resend.
            aliceTestClient.client.setDeviceKnown(aliceTestClient.userId, 'DEVICE_ID');
@@ -846,55 +820,33 @@ describe("megolm", function() {
                };
            });

-            return q.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                aliceTestClient.httpBackend.flush(),
+            // Grab the event that we'll need to resend
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const pendingEvents = room.getPendingEvents();
+            expect(pendingEvents.length).toEqual(1);
+            const unsentEvent = pendingEvents[0];
+
+            return Promise.all([
+                aliceTestClient.client.resendEvent(unsentEvent, room),
+
+                // the crypto stuff can take a while, so give the requests a whole second.
+                aliceTestClient.httpBackend.flushAllExpected({
+                    timeout: 1000,
+                }),
            ]);
        }).then(function() {
            expect(decrypted.type).toEqual('m.room.message');
            expect(decrypted.content.body).toEqual('test');
-        }).nodeify(done);
+        });
    });


    it('Alice should wait for device list to complete when sending a megolm message',
-    function(done) {
-        let p2pSession;
-        let inboundGroupSession;
-
+    function() {
        let downloadPromise;
        let sendPromise;

-        aliceTestClient.httpBackend.when(
-            'PUT', '/sendToDevice/m.room.encrypted/',
-        ).respond(200, function(path, content) {
-            const m = content.messages['@bob:xyz'].DEVICE_ID;
-            const ct = m.ciphertext[testSenderKey];
-            const decrypted = JSON.parse(p2pSession.decrypt(ct.type, ct.body));
-
-            expect(decrypted.type).toEqual('m.room_key');
-            inboundGroupSession = new Olm.InboundGroupSession();
-            inboundGroupSession.create(decrypted.content.session_key);
-            return {};
-        });
-
-        aliceTestClient.httpBackend.when(
-            'PUT', '/send/',
-        ).respond(200, function(path, content) {
-            const ct = content.ciphertext;
-            const r = inboundGroupSession.decrypt(ct);
-            console.log('Decrypted received megolm message', r);
-
-            expect(r.message_index).toEqual(0);
-            const decrypted = JSON.parse(r.plaintext);
-            expect(decrypted.type).toEqual('m.room.message');
-            expect(decrypted.content.body).toEqual('test');
-
-            return {
-                event_id: '$event_id',
-            };
-        });
-
+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
            return createOlmSession(testOlmAccount, aliceTestClient);
@@ -910,13 +862,10 @@ describe("megolm", function() {
            syncResponse.to_device = { events: [olmEvent] };

            aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush('/sync', 1);
+            return aliceTestClient.flushSync();
        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
-        }).then(function() {
-            console.log('Forcing alice to download our device keys');
-
            // this will block
+            console.log('Forcing alice to download our device keys');
            downloadPromise = aliceTestClient.client.downloadKeys(['@bob:xyz']);

            // so will this.
@@ -931,139 +880,17 @@ describe("megolm", function() {
                200, getTestKeysQueryResponse('@bob:xyz'),
            );

-            return aliceTestClient.httpBackend.flush();
+            return aliceTestClient.httpBackend.flushAllExpected();
        }).then(function() {
-            return q.all([downloadPromise, sendPromise]);
-        }).nodeify(done);
+            return Promise.all([downloadPromise, sendPromise]);
+        });
    });


-    it("We should not get confused by out-of-order device query responses",
-       () => {
-           // https://github.com/vector-im/riot-web/issues/3126
-           return aliceTestClient.start().then(() => {
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(
-                   200, getSyncResponse(['@bob:xyz', '@chris:abc']));
-               return aliceTestClient.httpBackend.flush('/sync', 1);
-           }).then(() => {
-               return testUtils.syncPromise(aliceTestClient.client);
-           }).then(() => {
-               // to make sure the initial device queries are flushed out, we
-               // attempt to send a message.
-
-               aliceTestClient.httpBackend.when('POST', '/keys/query').respond(
-                   200, {
-                       device_keys: {
-                           '@bob:xyz': {},
-                           '@chris:abc': {},
-                       },
-                   },
-               );
-
-               aliceTestClient.httpBackend.when('PUT', '/send/').respond(
-                   200, {event_id: '$event1'});
-
-               return q.all([
-                   aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
-                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
-                       () => aliceTestClient.httpBackend.flush('/send/', 1, 20),
-                   ),
-               ]);
-           }).then(() => {
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
-
-               // invalidate bob's and chris's device lists in separate syncs
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '2',
-                   device_lists: {
-                       changed: ['@bob:xyz'],
-                   },
-               });
-               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
-                   next_batch: '3',
-                   device_lists: {
-                       changed: ['@chris:abc'],
-                   },
-               });
-               return aliceTestClient.httpBackend.flush('/sync', 2);
-           }).then(() => {
-               return testUtils.syncPromise(aliceTestClient.client);
-           }).then(() => {
-               // check that we don't yet have a request for chris's devices.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@chris:abc': {},
-                   },
-                   token: '3',
-               }).respond(200, {
-                   device_keys: {'@chris:abc': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(0);
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               if (bobStat != 1 && bobStat != 2) {
-                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               // now add an expectation for a query for bob's devices, and let
-               // it complete.
-               aliceTestClient.httpBackend.when('POST', '/keys/query', {
-                   device_keys: {
-                       '@bob:xyz': {},
-                   },
-                   token: '2',
-               }).respond(200, {
-                   device_keys: {'@bob:xyz': {}},
-               });
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               expect(bobStat).toEqual(3);
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               // now let the query for chris's devices complete.
-               return aliceTestClient.httpBackend.flush('/keys/query', 1);
-           }).then((flushed) => {
-               expect(flushed).toEqual(1);
-
-               // wait for the client to stop processing the response
-               return aliceTestClient.client.downloadKeys(['@chris:abc']);
-           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-
-               expect(bobStat).toEqual(3);
-               expect(chrisStat).toEqual(3);
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
-           });
-       });
-
-    it("Alice exports megolm keys and imports them to a new device", function(done) {
+    it("Alice exports megolm keys and imports them to a new device", function() {
        let messageEncrypted;

+        aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
        return aliceTestClient.start().then(() => {
            // establish an olm session with alice
            return createOlmSession(testOlmAccount, aliceTestClient);
@@ -1104,9 +931,7 @@ describe("megolm", function() {
            };

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
@@ -1120,10 +945,10 @@ describe("megolm", function() {
            aliceTestClient = new TestClient(
                "@alice:localhost", "device2", "access_token2",
            );
-
-            aliceTestClient.client.importRoomKeys(exported);
-
-            return aliceTestClient.start();
+            return aliceTestClient.client.initCrypto().then(() => {
+                aliceTestClient.client.importRoomKeys(exported);
+                return aliceTestClient.start();
+            });
        }).then(function() {
            const syncResponse = {
                next_batch: 1,
@@ -1138,13 +963,11 @@ describe("megolm", function() {
            };

            aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
-            return aliceTestClient.httpBackend.flush("/sync", 1);
-        }).then(function() {
-            return testUtils.syncPromise(aliceTestClient.client);
+            return aliceTestClient.flushSync();
        }).then(function() {
            const room = aliceTestClient.client.getRoom(ROOM_ID);
            const event = room.getLiveTimeline().getEvents()[0];
            expect(event.getContent().body).toEqual('42');
-        }).nodeify(done);
+        });
    });
 });
@@ -1,295 +0,0 @@
-"use strict";
-const q = require("q");
-import expect from 'expect';
-
-/**
- * Construct a mock HTTP backend, heavily inspired by Angular.js.
- * @constructor
- */
-function HttpBackend() {
-    this.requests = [];
-    this.expectedRequests = [];
-    const self = this;
-    // the request function dependency that the SDK needs.
-    this.requestFn = function(opts, callback) {
-        const req = new Request(opts, callback);
-        console.log("HTTP backend received request: %s", req);
-        self.requests.push(req);
-
-        const abort = function() {
-            const idx = self.requests.indexOf(req);
-            if (idx >= 0) {
-                console.log("Aborting HTTP request: %s %s", opts.method,
-                            opts.uri);
-                self.requests.splice(idx, 1);
-                req.callback("aborted");
-            }
-        };
-
-        return {
-            abort: abort,
-        };
-    };
-}
-HttpBackend.prototype = {
-    /**
-     * Respond to all of the requests (flush the queue).
-     * @param {string} path The path to flush (optional) default: all.
-     * @param {integer} numToFlush The number of things to flush (optional), default: all.
-     * @param {integer=} waitTime  The time (in ms) to wait for a request to happen.
-     *    default: 5
-     *
-     * @return {Promise} resolves when there is nothing left to flush, with the
-     *    number of requests flushed
-     */
-    flush: function(path, numToFlush, waitTime) {
-        const defer = q.defer();
-        const self = this;
-        let flushed = 0;
-        let triedWaiting = false;
-        if (waitTime === undefined) {
-            waitTime = 5;
-        }
-        console.log(
-            "HTTP backend flushing... (path=%s numToFlush=%s waitTime=%s)",
-            path, numToFlush, waitTime,
-        );
-        const tryFlush = function() {
-            // if there's more real requests and more expected requests, flush 'em.
-            console.log(
-                "  trying to flush queue => reqs=[%s] expected=[%s]",
-                self.requests, self.expectedRequests.map((er) => er.path),
-            );
-            if (self._takeFromQueue(path)) {
-                // try again on the next tick.
-                flushed += 1;
-                if (numToFlush && flushed === numToFlush) {
-                    console.log("  Flushed assigned amount: %s", numToFlush);
-                    defer.resolve(flushed);
-                } else {
-                    console.log("  flushed. Trying for more.");
-                    setTimeout(tryFlush, 0);
-                }
-            } else if (flushed === 0 && !triedWaiting) {
-                // we may not have made the request yet, wait a generous amount of
-                // time before giving up.
-                setTimeout(tryFlush, waitTime);
-                triedWaiting = true;
-            } else {
-                console.log("  no more flushes.");
-                defer.resolve(flushed);
-            }
-        };
-
-        setTimeout(tryFlush, 0);
-
-        return defer.promise;
-    },
-
-    /**
-     * Attempts to resolve requests/expected requests.
-     * @param {string} path The path to flush (optional) default: all.
-     * @return {boolean} true if something was resolved.
-     */
-    _takeFromQueue: function(path) {
-        let req = null;
-        let i;
-        let j;
-        let matchingReq = null;
-        let expectedReq = null;
-        let testResponse = null;
-        for (i = 0; i < this.requests.length; i++) {
-            req = this.requests[i];
-            for (j = 0; j < this.expectedRequests.length; j++) {
-                expectedReq = this.expectedRequests[j];
-                if (path && path !== expectedReq.path) {
-                    continue;
-                }
-                if (expectedReq.method === req.method &&
-                        req.path.indexOf(expectedReq.path) !== -1) {
-                    if (!expectedReq.data || (JSON.stringify(expectedReq.data) ===
-                            JSON.stringify(req.data))) {
-                        matchingReq = expectedReq;
-                        this.expectedRequests.splice(j, 1);
-                        break;
-                    }
-                }
-            }
-
-            if (matchingReq) {
-                // remove from request queue
-                this.requests.splice(i, 1);
-                i--;
-
-                for (j = 0; j < matchingReq.checks.length; j++) {
-                    matchingReq.checks[j](req);
-                }
-                testResponse = matchingReq.response;
-                console.log("    responding to %s", matchingReq.path);
-                let body = testResponse.body;
-                if (Object.prototype.toString.call(body) == "[object Function]") {
-                    body = body(req.path, req.data);
-                }
-                req.callback(
-                    testResponse.err, testResponse.response, body,
-                );
-                matchingReq = null;
-            }
-        }
-        if (testResponse) {  // flushed something
-            return true;
-        }
-        return false;
-    },
-
-    /**
-     * Makes sure that the SDK hasn't sent any more requests to the backend.
-     */
-    verifyNoOutstandingRequests: function() {
-        const firstOutstandingReq = this.requests[0] || {};
-        expect(this.requests.length).toEqual(0,
-            "Expected no more HTTP requests but received request to " +
-            firstOutstandingReq.path,
-        );
-    },
-
-    /**
-     * Makes sure that the test doesn't have any unresolved requests.
-     */
-    verifyNoOutstandingExpectation: function() {
-        const firstOutstandingExpectation = this.expectedRequests[0] || {};
-        expect(this.expectedRequests.length).toEqual(0,
-            "Expected to see HTTP request for " + firstOutstandingExpectation.path,
-        );
-    },
-
-    /**
-     * Create an expected request.
-     * @param {string} method The HTTP method
-     * @param {string} path The path (which can be partial)
-     * @param {Object} data The expected data.
-     * @return {Request} An expected request.
-     */
-    when: function(method, path, data) {
-        const pendingReq = new ExpectedRequest(method, path, data);
-        this.expectedRequests.push(pendingReq);
-        return pendingReq;
-    },
-};
-
-/**
- * Represents the expectation of a request.
- *
- * <p>Includes the conditions to be matched against, the checks to be made,
- * and the response to be returned.
- *
- * @constructor
- * @param {string} method
- * @param {string} path
- * @param {object?} data
- */
-function ExpectedRequest(method, path, data) {
-    this.method = method;
-    this.path = path;
-    this.data = data;
-    this.response = null;
-    this.checks = [];
-}
-
-ExpectedRequest.prototype = {
-    /**
-     * Execute a check when this request has been satisfied.
-     * @param {Function} fn The function to execute.
-     * @return {Request} for chaining calls.
-     */
-    check: function(fn) {
-        this.checks.push(fn);
-        return this;
-    },
-
-    /**
-     * Respond with the given data when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Object|Function} data The HTTP JSON body. If this is a function,
-     * it will be invoked when the JSON body is required (which should be returned).
-     */
-    respond: function(code, data) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {},
-            },
-            body: data,
-            err: null,
-        };
-    },
-
-    /**
-     * Fail with an Error when this request is satisfied.
-     * @param {Number} code The HTTP status code.
-     * @param {Error} err The error to throw (e.g. Network Error)
-     */
-    fail: function(code, err) {
-        this.response = {
-            response: {
-                statusCode: code,
-                headers: {},
-            },
-            body: null,
-            err: err,
-        };
-    },
-};
-
-/**
- * Represents a request made by the app.
- *
- * @constructor
- * @param {object} opts opts passed to request()
- * @param {function} callback
- */
-function Request(opts, callback) {
-    this.opts = opts;
-    this.callback = callback;
-
-    Object.defineProperty(this, 'method', {
-        get: function() {
-            return opts.method;
-        },
-    });
-
-    Object.defineProperty(this, 'path', {
-        get: function() {
-            return opts.uri;
-        },
-    });
-
-    Object.defineProperty(this, 'data', {
-        get: function() {
-            return opts.body;
-        },
-    });
-
-    Object.defineProperty(this, 'queryParams', {
-        get: function() {
-            return opts.qs;
-        },
-    });
-
-    Object.defineProperty(this, 'headers', {
-        get: function() {
-            return opts.headers || {};
-        },
-    });
-}
-
-Request.prototype = {
-    toString: function() {
-        return this.method + " " + this.path;
-    },
-};
-
-/**
- * The HttpBackend class.
- */
-module.exports = HttpBackend;
@@ -0,0 +1,24 @@
+/*
+Copyright 2017 Vector creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// try to load the olm library.
+
+try {
+    global.Olm = require('olm');
+    console.log('loaded libolm');
+} catch (e) {
+    console.warn("unable to run crypto tests: libolm not available");
+}
@@ -1,27 +1,43 @@
 "use strict";
 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';

-const sdk = require("..");
+// load olm before the sdk if possible
+import './olm-loader';
+
+import sdk from '..';
 const MatrixEvent = sdk.MatrixEvent;

 /**
 * Return a promise that is resolved when the client next emits a
 * SYNCING event.
 * @param {Object} client The client
+ * @param {Number=} count Number of syncs to wait for (default 1)
 * @return {Promise} Resolves once the client has emitted a SYNCING event
 */
-module.exports.syncPromise = function(client) {
-    const def = q.defer();
-    const cb = (state) => {
-        if (state == 'SYNCING') {
-            def.resolve();
-        } else {
-            client.once('sync', cb);
-        }
-    };
-    client.once('sync', cb);
-    return def.promise;
+module.exports.syncPromise = function(client, count) {
+    if (count === undefined) {
+        count = 1;
+    }
+    if (count <= 0) {
+        return Promise.resolve();
+    }
+
+    const p = new Promise((resolve, reject) => {
+        const cb = (state) => {
+            console.log(`${Date.now()} syncPromise(${count}): ${state}`);
+            if (state == 'SYNCING') {
+                resolve();
+            } else {
+                client.once('sync', cb);
+            }
+        };
+        client.once('sync', cb);
+    });
+
+    return p.then(() => {
+        return module.exports.syncPromise(client, count-1);
+    });
 };

 /**
@@ -178,35 +194,6 @@ module.exports.mkMessage = function(opts) {
 };


-/**
- * make the test fail, with the given exception
- *
- * <p>This is useful for use with integration tests which use asyncronous
- * methods: it can be added as a 'catch' handler in a promise chain.
- *
- * @param {Error} err   exception to be reported
- *
- * @deprecated
- * It turns out there are easier ways of doing this. Just use nodeify():
- *
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).nodeify(done);
- * });
- *
- * @example
- * it("should not throw", function(done) {
- *    asynchronousMethod().then(function() {
- *       // some tests
- *    }).catch(utils.failTest).done(done);
- * });
- */
-module.exports.failTest = function(err) {
-    expect(true).toBe(false, "Testfunc threw: " + err.stack);
-};
-
-
 /**
 * A mock implementation of webstorage
 *
@@ -232,3 +219,25 @@ module.exports.MockStorageApi.prototype = {
        delete this.data[k];
    },
 };
+
+
+/**
+ * If an event is being decrypted, wait for it to finish being decrypted.
+ *
+ * @param {MatrixEvent} event
+ * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
+ */
+module.exports.awaitDecryption = function(event) {
+    if (!event.isBeingDecrypted()) {
+        return Promise.resolve(event);
+    }
+
+    console.log(`${Date.now()} event ${event.getId()} is being decrypted; waiting`);
+
+    return new Promise((resolve, reject) => {
+        event.once('Event.decrypted', (ev) => {
+            console.log(`${Date.now()} event ${event.getId()} now decrypted`);
+            resolve(ev);
+        });
+    });
+};
@@ -0,0 +1,670 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+import 'source-map-support/register';
+import Promise from 'bluebird';
+const sdk = require("../..");
+const utils = require("../test-utils");
+
+const AutoDiscovery = sdk.AutoDiscovery;
+
+import expect from 'expect';
+import MockHttpBackend from "matrix-mock-request";
+
+
+describe("AutoDiscovery", function() {
+    let httpBackend = null;
+
+    beforeEach(function() {
+        utils.beforeEach(this); // eslint-disable-line no-invalid-this
+        httpBackend = new MockHttpBackend();
+        sdk.request(httpBackend.requestFn);
+    });
+
+    it("should throw an error when no domain is specified", function() {
+        return Promise.all([
+            AutoDiscovery.findClientConfig(/* no args */).then(() => {
+                throw new Error("Expected a failure, not success with no args");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig("").then(() => {
+                throw new Error("Expected a failure, not success with an empty string");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig(null).then(() => {
+                throw new Error("Expected a failure, not success with null");
+            }, () => {
+                return true;
+            }),
+
+            AutoDiscovery.findClientConfig(true).then(() => {
+                throw new Error("Expected a failure, not success with a non-string");
+            }, () => {
+                return true;
+            }),
+        ]);
+    });
+
+    it("should return PROMPT when .well-known 404s", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(404, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns a 500 error", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(500, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns a 400 error", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(400, {});
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns an empty body", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "");
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known returns not-JSON", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, "abc");
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
+        "m.homeserver (empty string)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_PROMPT when .well-known does not have a base_url for " +
+        "m.homeserver (no property)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {},
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_PROMPT",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (disallowed scheme)", function() {
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "mxc://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 404)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(404, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 500)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(500, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when .well-known has an invalid base_url for " +
+        "m.homeserver (verification failure: 200 but wrong content)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            not_matrix_versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid homeserver discovery response",
+                        base_url: null,
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS when .well-known has a verifiably accurate base_url for " +
+        "m.homeserver", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri).toEqual("https://example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS with the right homeserver URL", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when the identity server configuration is wrong " +
+        "(missing base_url)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                not_base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when the identity server configuration is wrong " +
+        "(empty base_url)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when the identity server configuration is wrong " +
+        "(validation error: 404)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").respond(404, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+
+                        // We still expect the base_url to be here for debugging purposes.
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return FAIL_ERROR when the identity server configuration is wrong " +
+        "(validation error: 500)", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").respond(500, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+
+                        // We still expect the base_url to be here for debugging purposes
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "FAIL_ERROR",
+                        error: "Invalid identity server discovery response",
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS when the identity server configuration is " +
+        "verifiably accurate", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://identity.example.org/_matrix/identity/api/v1");
+        }).respond(200, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://identity.example.org",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    it("should return SUCCESS and preserve non-standard keys from the " +
+        ".well-known response", function() {
+        httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://chat.example.org/_matrix/client/versions");
+        }).respond(200, {
+            versions: ["r0.0.1"],
+        });
+        httpBackend.when("GET", "/_matrix/identity/api/v1").check((req) => {
+            expect(req.opts.uri)
+                .toEqual("https://identity.example.org/_matrix/identity/api/v1");
+        }).respond(200, {});
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                // Note: we also expect this test to trim the trailing slash
+                base_url: "https://chat.example.org/",
+            },
+            "m.identity_server": {
+                base_url: "https://identity.example.org",
+            },
+            "org.example.custom.property": {
+                cupcakes: "yes",
+            },
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://chat.example.org",
+                    },
+                    "m.identity_server": {
+                        state: "SUCCESS",
+                        error: null,
+                        base_url: "https://identity.example.org",
+                    },
+                    "org.example.custom.property": {
+                        cupcakes: "yes",
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+});
@@ -1,20 +1,366 @@
-
-"use strict";
 import 'source-map-support/register';

-const sdk = require("../..");
-let Crypto;
-if (sdk.CRYPTO_ENABLED) {
-    Crypto = require("../../lib/crypto");
-}
+import '../olm-loader';

+import Crypto from '../../lib/crypto';
 import expect from 'expect';

+import WebStorageSessionStore from '../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../MockStorageApi';
+import TestClient from '../TestClient';
+import {MatrixEvent} from '../../lib/models/event';
+import Room from '../../lib/models/room';
+import olmlib from '../../lib/crypto/olmlib';
+import lolex from 'lolex';
+
+const EventEmitter = require("events").EventEmitter;
+
+const sdk = require("../..");
+
+const Olm = global.Olm;
+
 describe("Crypto", function() {
    if (!sdk.CRYPTO_ENABLED) {
        return;
    }
+
+    beforeEach(function(done) {
+        Olm.init().then(done);
+    });
+
    it("Crypto exposes the correct olm library version", function() {
-        expect(Crypto.getOlmVersion()[0]).toEqual(2);
+        expect(Crypto.getOlmVersion()[0]).toEqual(3);
+    });
+
+
+    describe('Session management', function() {
+        const otkResponse = {
+            one_time_keys: {
+                '@alice:home.server': {
+                    aliceDevice: {
+                        'signed_curve25519:FLIBBLE': {
+                            key: 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI',
+                            signatures: {
+                                '@alice:home.server': {
+                                    'ed25519:aliceDevice': 'totally a valid signature',
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        };
+        let crypto;
+        let mockBaseApis;
+        let mockRoomList;
+
+        let fakeEmitter;
+
+        beforeEach(async function() {
+            const mockStorage = new MockStorageApi();
+            const sessionStore = new WebStorageSessionStore(mockStorage);
+            const cryptoStore = new MemoryCryptoStore(mockStorage);
+
+            cryptoStore.storeEndToEndDeviceData({
+                devices: {
+                    '@bob:home.server': {
+                        'BOBDEVICE': {
+                            keys: {
+                                'curve25519:BOBDEVICE': 'this is a key',
+                            },
+                        },
+                    },
+                },
+                trackingStatus: {},
+            });
+
+            mockBaseApis = {
+                sendToDevice: expect.createSpy(),
+                getKeyBackupVersion: expect.createSpy(),
+                isGuest: expect.createSpy(),
+            };
+            mockRoomList = {};
+
+            fakeEmitter = new EventEmitter();
+
+            crypto = new Crypto(
+                mockBaseApis,
+                sessionStore,
+                "@alice:home.server",
+                "FLIBBLE",
+                sessionStore,
+                cryptoStore,
+                mockRoomList,
+            );
+            crypto.registerEventHandlers(fakeEmitter);
+            await crypto.init();
+        });
+
+        afterEach(async function() {
+            await crypto.stop();
+        });
+
+        it("restarts wedged Olm sessions", async function() {
+            const prom = new Promise((resolve) => {
+                mockBaseApis.claimOneTimeKeys = function() {
+                    resolve();
+                    return otkResponse;
+                };
+            });
+
+            fakeEmitter.emit('toDeviceEvent', {
+                getType: expect.createSpy().andReturn('m.room.message'),
+                getContent: expect.createSpy().andReturn({
+                    msgtype: 'm.bad.encrypted',
+                }),
+                getWireContent: expect.createSpy().andReturn({
+                    algorithm: 'm.olm.v1.curve25519-aes-sha2',
+                    sender_key: 'this is a key',
+                }),
+                getSender: expect.createSpy().andReturn('@bob:home.server'),
+            });
+
+            await prom;
+        });
+    });
+
+    describe('Key requests', function() {
+        let aliceClient;
+        let bobClient;
+
+        beforeEach(async function() {
+            aliceClient = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            bobClient = (new TestClient(
+                "@bob:example.com", "bobdevice",
+            )).client;
+            await aliceClient.initCrypto();
+            await bobClient.initCrypto();
+        });
+
+        afterEach(async function() {
+            aliceClient.stopClient();
+            bobClient.stopClient();
+        });
+
+        it(
+            "does not cancel keyshare requests if some messages are not decrypted",
+            async function() {
+                function awaitEvent(emitter, event) {
+                    return new Promise((resolve, reject) => {
+                        emitter.once(event, (result) => {
+                            resolve(result);
+                        });
+                    });
+                }
+
+                async function keyshareEventForEvent(event, index) {
+                    const eventContent = event.getWireContent();
+                    const key = await aliceClient._crypto._olmDevice
+                        .getInboundGroupSessionKey(
+                            roomId, eventContent.sender_key, eventContent.session_id,
+                            index,
+                        );
+                    const ksEvent = new MatrixEvent({
+                        type: "m.forwarded_room_key",
+                        sender: "@alice:example.com",
+                        content: {
+                            algorithm: olmlib.MEGOLM_ALGORITHM,
+                            room_id: roomId,
+                            sender_key: eventContent.sender_key,
+                            sender_claimed_ed25519_key: key.sender_claimed_ed25519_key,
+                            session_id: eventContent.session_id,
+                            session_key: key.key,
+                            chain_index: key.chain_index,
+                            forwarding_curve25519_key_chain:
+                            key.forwarding_curve_key_chain,
+                        },
+                    });
+                    // make onRoomKeyEvent think this was an encrypted event
+                    ksEvent._senderCurve25519Key = "akey";
+                    return ksEvent;
+                }
+
+                const encryptionCfg = {
+                    "algorithm": "m.megolm.v1.aes-sha2",
+                };
+                const roomId = "!someroom";
+                const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+                const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+                aliceClient.store.storeRoom(aliceRoom);
+                bobClient.store.storeRoom(bobRoom);
+                await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+                await bobClient.setRoomEncryption(roomId, encryptionCfg);
+                const events = [
+                    new MatrixEvent({
+                        type: "m.room.message",
+                        sender: "@alice:example.com",
+                        room_id: roomId,
+                        event_id: "$1",
+                        content: {
+                            msgtype: "m.text",
+                            body: "1",
+                        },
+                    }),
+                    new MatrixEvent({
+                        type: "m.room.message",
+                        sender: "@alice:example.com",
+                        room_id: roomId,
+                        event_id: "$2",
+                        content: {
+                            msgtype: "m.text",
+                            body: "2",
+                        },
+                    }),
+                ];
+                await Promise.all(events.map(async (event) => {
+                    // alice encrypts each event, and then bob tries to decrypt
+                    // them without any keys, so that they'll be in pending
+                    await aliceClient._crypto.encryptEvent(event, aliceRoom);
+                    event._clearEvent = {};
+                    event._senderCurve25519Key = null;
+                    event._claimedEd25519Key = null;
+                    try {
+                        await bobClient._crypto.decryptEvent(event);
+                    } catch (e) {
+                        // we expect this to fail because we don't have the
+                        // decryption keys yet
+                    }
+                }));
+
+                const bobDecryptor = bobClient._crypto._getRoomDecryptor(
+                    roomId, olmlib.MEGOLM_ALGORITHM,
+                );
+
+                let eventPromise = Promise.all(events.map((ev) => {
+                    return awaitEvent(ev, "Event.decrypted");
+                }));
+
+                // keyshare the session key starting at the second message, so
+                // the first message can't be decrypted yet, but the second one
+                // can
+                let ksEvent = await keyshareEventForEvent(events[1], 1);
+                await bobDecryptor.onRoomKeyEvent(ksEvent);
+                await eventPromise;
+                expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
+                expect(events[1].getContent().msgtype).toNotBe("m.bad.encrypted");
+
+                const cryptoStore = bobClient._cryptoStore;
+                const eventContent = events[0].getWireContent();
+                const senderKey = eventContent.sender_key;
+                const sessionId = eventContent.session_id;
+                const roomKeyRequestBody = {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    room_id: roomId,
+                    sender_key: senderKey,
+                    session_id: sessionId,
+                };
+                // the room key request should still be there, since we haven't
+                // decrypted everything
+                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                    .toExist();
+
+                // keyshare the session key starting at the first message, so
+                // that it can now be decrypted
+                eventPromise = awaitEvent(events[0], "Event.decrypted");
+                ksEvent = await keyshareEventForEvent(events[0], 0);
+                await bobDecryptor.onRoomKeyEvent(ksEvent);
+                await eventPromise;
+                expect(events[0].getContent().msgtype).toNotBe("m.bad.encrypted");
+                // the room key request should be gone since we've now decypted everything
+                expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                    .toNotExist();
+            },
+        );
+
+        it("creates a new keyshare request if we request a keyshare", async function() {
+            // make sure that cancelAndResend... creates a new keyshare request
+            // if there wasn't an already-existing one
+            const event = new MatrixEvent({
+                sender: "@bob:example.com",
+                room_id: "!someroom",
+                content: {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    session_id: "sessionid",
+                    sender_key: "senderkey",
+                },
+            });
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            const cryptoStore = aliceClient._cryptoStore;
+            const roomKeyRequestBody = {
+                algorithm: olmlib.MEGOLM_ALGORITHM,
+                room_id: "!someroom",
+                session_id: "sessionid",
+                sender_key: "senderkey",
+            };
+            expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
+                .toExist();
+        });
+
+        it("uses a new txnid for re-requesting keys", async function() {
+            const event = new MatrixEvent({
+                sender: "@bob:example.com",
+                room_id: "!someroom",
+                content: {
+                    algorithm: olmlib.MEGOLM_ALGORITHM,
+                    session_id: "sessionid",
+                    sender_key: "senderkey",
+                },
+            });
+            /* return a promise and a function. When the function is called,
+             * the promise will be resolved.
+             */
+            function awaitFunctionCall() {
+                let func;
+                const promise = new Promise((resolve, reject) => {
+                    func = function(...args) {
+                        resolve(args);
+                        return new Promise((resolve, reject) => {
+                            // give us some time to process the result before
+                            // continuing
+                            global.setTimeout(resolve, 1);
+                        });
+                    };
+                });
+                return {func, promise};
+            }
+
+            aliceClient.startClient();
+
+            const clock = lolex.install();
+
+            try {
+                let promise;
+                // make a room key request, and record the transaction ID for the
+                // sendToDevice call
+                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
+                await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+                clock.runToLast();
+                let args = await promise;
+                const txnId = args[2];
+                clock.runToLast();
+
+                // give the room key request manager time to update the state
+                // of the request
+                await Promise.resolve();
+
+                // cancel and resend the room key request
+                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
+                await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+                clock.runToLast();
+                // the first call to sendToDevice will be the cancellation
+                args = await promise;
+                // the second call to sendToDevice will be the key request
+                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
+                clock.runToLast();
+                args = await promise;
+                clock.runToLast();
+                expect(args[2]).toNotBe(txnId);
+            } finally {
+                clock.uninstall();
+            }
+        });
    });
 });
@@ -1,11 +1,29 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import DeviceList from '../../../lib/crypto/DeviceList';
 import MockStorageApi from '../../MockStorageApi';
 import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
 import testUtils from '../../test-utils';
 import utils from '../../../lib/utils';

 import expect from 'expect';
-import q from 'q';
+import Promise from 'bluebird';

 const signedDeviceList = {
    "failures": {},
@@ -40,13 +58,24 @@ const signedDeviceList = {
 describe('DeviceList', function() {
    let downloadSpy;
    let sessionStore;
+    let cryptoStore;
+    let deviceLists = [];

    beforeEach(function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this

+        deviceLists = [];
+
        downloadSpy = expect.createSpy();
        const mockStorage = new MockStorageApi();
        sessionStore = new WebStorageSessionStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore();
+    });
+
+    afterEach(function() {
+        for (const dl of deviceLists) {
+            dl.stop();
+        }
    });

    function createTestDeviceList() {
@@ -56,7 +85,9 @@ describe('DeviceList', function() {
        const mockOlm = {
            verifySignature: function(key, message, signature) {},
        };
-        return new DeviceList(baseApis, sessionStore, mockOlm);
+        const dl = new DeviceList(baseApis, cryptoStore, sessionStore, mockOlm);
+        deviceLists.push(dl);
+        return dl;
    }

    it("should successfully download and store device keys", function() {
@@ -64,7 +95,7 @@ describe('DeviceList', function() {

        dl.startTrackingDeviceList('@test1:sw1v.org');

-        const queryDefer1 = q.defer();
+        const queryDefer1 = Promise.defer();
        downloadSpy.andReturn(queryDefer1.promise);

        const prom1 = dl.refreshOutdatedDeviceLists();
@@ -72,7 +103,7 @@ describe('DeviceList', function() {
        queryDefer1.resolve(utils.deepCopy(signedDeviceList));

        return prom1.then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
        });
    });
@@ -83,7 +114,7 @@ describe('DeviceList', function() {

        dl.startTrackingDeviceList('@test1:sw1v.org');

-        const queryDefer1 = q.defer();
+        const queryDefer1 = Promise.defer();
        downloadSpy.andReturn(queryDefer1.promise);

        const prom1 = dl.refreshOutdatedDeviceLists();
@@ -91,26 +122,27 @@ describe('DeviceList', function() {
        downloadSpy.reset();

        // outdated notif arrives while the request is in flight.
-        const queryDefer2 = q.defer();
+        const queryDefer2 = Promise.defer();
        downloadSpy.andReturn(queryDefer2.promise);

        dl.invalidateUserDeviceList('@test1:sw1v.org');
        dl.refreshOutdatedDeviceLists();

-        // the first request completes
-        queryDefer1.resolve({
-            device_keys: {
-                '@test1:sw1v.org': {},
-            },
-        });
-
-        return prom1.then(() => {
+        dl.saveIfDirty().then(() => {
+            // the first request completes
+            queryDefer1.resolve({
+                device_keys: {
+                    '@test1:sw1v.org': {},
+                },
+            });
+            return prom1;
+        }).then(() => {
            // uh-oh; user restarts before second request completes. The new instance
            // should know we never got a complete device list.
            console.log("Creating new devicelist to simulate app reload");
            downloadSpy.reset();
            const dl2 = createTestDeviceList();
-            const queryDefer3 = q.defer();
+            const queryDefer3 = Promise.defer();
            downloadSpy.andReturn(queryDefer3.promise);

            const prom3 = dl2.refreshOutdatedDeviceLists();
@@ -121,7 +153,7 @@ describe('DeviceList', function() {
            // allow promise chain to complete
            return prom3;
        }).then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
        });
    });
@@ -0,0 +1,352 @@
+import '../../../olm-loader';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import sdk from '../../../..';
+import algorithms from '../../../../lib/crypto/algorithms';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+import Crypto from '../../../../lib/crypto';
+
+const MatrixEvent = sdk.MatrixEvent;
+const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
+const MegolmEncryption = algorithms.ENCRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
+
+const ROOM_ID = '!ROOM:ID';
+
+const Olm = global.Olm;
+
+describe("MegolmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let megolmDecryption;
+    let mockOlmLib;
+    let mockCrypto;
+    let mockBaseApis;
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        await Olm.init();
+
+        mockCrypto = testUtils.mock(Crypto, 'Crypto');
+        mockBaseApis = {};
+
+        const mockStorage = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(mockStorage);
+        const cryptoStore = new MemoryCryptoStore(mockStorage);
+
+        const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+
+        megolmDecryption = new MegolmDecryption({
+            userId: '@user:id',
+            crypto: mockCrypto,
+            olmDevice: olmDevice,
+            baseApis: mockBaseApis,
+            roomId: ROOM_ID,
+        });
+
+
+        // we stub out the olm encryption bits
+        mockOlmLib = {};
+        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.encryptMessageForDevice =
+            expect.createSpy().andReturn(Promise.resolve());
+        megolmDecryption.olmlib = mockOlmLib;
+    });
+
+    describe('receives some keys:', function() {
+        let groupSession;
+        beforeEach(async function() {
+            groupSession = new global.Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // construct a fake decrypted key event via the use of a mocked
+            // 'crypto' implementation.
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+            });
+            const decryptedData = {
+                clearEvent: {
+                    type: 'm.room_key',
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        room_id: ROOM_ID,
+                        session_id: groupSession.session_id(),
+                        session_key: groupSession.session_key(),
+                    },
+                },
+                senderCurve25519Key: "SENDER_CURVE25519",
+                claimedEd25519Key: "SENDER_ED25519",
+            };
+
+            const mockCrypto = {
+                decryptEvent: function() {
+                    return Promise.resolve(decryptedData);
+                },
+            };
+
+            await event.attemptDecryption(mockCrypto).then(() => {
+                megolmDecryption.onRoomKeyEvent(event);
+            });
+        });
+
+        it('can decrypt an event', function() {
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                    ciphertext: groupSession.encrypt(JSON.stringify({
+                        room_id: ROOM_ID,
+                        content: 'testytest',
+                    })),
+                },
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
+            });
+        });
+
+        it('can respond to a key request event', function() {
+            const keyRequest = {
+                userId: '@alice:foo',
+                deviceId: 'alidevice',
+                requestBody: {
+                    room_id: ROOM_ID,
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: groupSession.session_id(),
+                },
+            };
+
+            return megolmDecryption.hasKeysForKeyRequest(
+                keyRequest,
+            ).then((hasKeys) => {
+                expect(hasKeys).toBe(true);
+
+                // set up some pre-conditions for the share call
+                const deviceInfo = {};
+                mockCrypto.getStoredDevice.andReturn(deviceInfo);
+
+                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
+                    Promise.resolve({'@alice:foo': {'alidevice': {
+                        sessionId: 'alisession',
+                    }}}),
+                );
+
+                const awaitEncryptForDevice = new Promise((res, rej) => {
+                    mockOlmLib.encryptMessageForDevice.andCall(() => {
+                        res();
+                        return Promise.resolve();
+                    });
+                });
+
+                mockBaseApis.sendToDevice = expect.createSpy();
+
+                // do the share
+                megolmDecryption.shareKeysWithDevice(keyRequest);
+
+                // it's asynchronous, so we have to wait a bit
+                return awaitEncryptForDevice;
+            }).then(() => {
+                // check that it called encryptMessageForDevice with
+                // appropriate args.
+                expect(mockOlmLib.encryptMessageForDevice.calls.length)
+                    .toEqual(1);
+
+                const call = mockOlmLib.encryptMessageForDevice.calls[0];
+                const payload = call.arguments[6];
+
+                expect(payload.type).toEqual("m.forwarded_room_key");
+                expect(payload.content).toInclude({
+                    sender_key: "SENDER_CURVE25519",
+                    sender_claimed_ed25519_key: "SENDER_ED25519",
+                    session_id: groupSession.session_id(),
+                    chain_index: 0,
+                    forwarding_curve25519_key_chain: [],
+                });
+                expect(payload.content.session_key).toExist();
+            });
+        });
+
+        it("can detect replay attacks", function() {
+            // trying to decrypt two different messages (marked by different
+            // event IDs or timestamps) using the same (sender key, session id,
+            // message index) triple should result in an exception being thrown
+            // as it should be detected as a replay attack.
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event1 = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            const successHandler = expect.createSpy();
+            const failureHandler = expect.createSpy()
+                .andCall((err) => {
+                    expect(err.toString()).toMatch(
+                        /Duplicate message index, possible replay attack/,
+                    );
+                });
+
+            return megolmDecryption.decryptEvent(event1).then((res) => {
+                const event2 = new MatrixEvent({
+                    type: 'm.room.encrypted',
+                    room_id: ROOM_ID,
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        sender_key: "SENDER_CURVE25519",
+                        session_id: sessionId,
+                        ciphertext: cipherText,
+                    },
+                    event_id: "$event2",
+                    origin_server_ts: 1507754149000,
+                });
+
+                return megolmDecryption.decryptEvent(event2);
+            }).then(
+                successHandler,
+                failureHandler,
+            ).then(() => {
+                expect(successHandler).toNotHaveBeenCalled();
+                expect(failureHandler).toHaveBeenCalled();
+            });
+        });
+
+        it("allows re-decryption of the same event", function() {
+            // in contrast with the previous test, if the event ID and
+            // timestamp are the same, then it should not be considered a
+            // replay attack
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                return megolmDecryption.decryptEvent(event);
+                // test is successful if no exception is thrown
+            });
+        });
+
+        it("re-uses sessions for sequential messages", async function() {
+            const mockStorage = new MockStorageApi();
+            const sessionStore = new WebStorageSessionStore(mockStorage);
+            const cryptoStore = new MemoryCryptoStore(mockStorage);
+
+            const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+            olmDevice.verifySignature = expect.createSpy();
+            await olmDevice.init();
+
+            mockBaseApis.claimOneTimeKeys = expect.createSpy().andReturn(Promise.resolve({
+                one_time_keys: {
+                    '@alice:home.server': {
+                        aliceDevice: {
+                            'signed_curve25519:flooble': {
+                                key: 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI',
+                                signatures: {
+                                    '@alice:home.server': {
+                                        'ed25519:aliceDevice': 'totally valid',
+                                    },
+                                },
+                            },
+                        },
+                    },
+                },
+            }));
+            mockBaseApis.sendToDevice = expect.createSpy().andReturn(Promise.resolve());
+
+            mockCrypto.downloadKeys.andReturn(Promise.resolve({
+                '@alice:home.server': {
+                    aliceDevice: {
+                        deviceId: 'aliceDevice',
+                        isBlocked: expect.createSpy().andReturn(false),
+                        isUnverified: expect.createSpy().andReturn(false),
+                        getIdentityKey: expect.createSpy().andReturn(
+                            'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE',
+                        ),
+                        getFingerprint: expect.createSpy().andReturn(''),
+                    },
+                },
+            }));
+
+            const megolmEncryption = new MegolmEncryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: mockBaseApis,
+                roomId: ROOM_ID,
+                config: {
+                    rotation_period_ms: 9999999999999,
+                },
+            });
+            const mockRoom = {
+                getEncryptionTargetMembers: expect.createSpy().andReturn(
+                    [{userId: "@alice:home.server"}],
+                ),
+                getBlacklistUnverifiedDevices: expect.createSpy().andReturn(false),
+            };
+            const ct1 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
+                body: "Some text",
+            });
+            expect(mockRoom.getEncryptionTargetMembers).toHaveBeenCalled();
+
+            // this should have claimed a key for alice as it's starting a new session
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            );
+            expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
+                ['@alice:home.server'], false,
+            );
+            expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            );
+
+            mockBaseApis.claimOneTimeKeys.reset();
+
+            const ct2 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
+                body: "Some more text",
+            });
+
+            // this should *not* have claimed a key as it should be using the same session
+            expect(mockBaseApis.claimOneTimeKeys).toNotHaveBeenCalled();
+
+            // likewise they should show the same session ID
+            expect(ct2.session_id).toEqual(ct1.session_id);
+        });
+    });
+});
@@ -0,0 +1,86 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../../olm-loader';
+
+import expect from 'expect';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+
+function makeOlmDevice() {
+    const mockStorage = new MockStorageApi();
+    const sessionStore = new WebStorageSessionStore(mockStorage);
+    const cryptoStore = new MemoryCryptoStore(mockStorage);
+    const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    return olmDevice;
+}
+
+async function setupSession(initiator, opponent) {
+    await opponent.generateOneTimeKeys(1);
+    const keys = await opponent.getOneTimeKeys();
+    const firstKey = Object.values(keys['curve25519'])[0];
+
+    const sid = await initiator.createOutboundSession(
+        opponent.deviceCurve25519Key, firstKey,
+    );
+    return sid;
+}
+
+describe("OlmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let aliceOlmDevice;
+    let bobOlmDevice;
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        await global.Olm.init();
+
+        aliceOlmDevice = makeOlmDevice();
+        bobOlmDevice = makeOlmDevice();
+        await aliceOlmDevice.init();
+        await bobOlmDevice.init();
+    });
+
+    describe('olm', function() {
+        it("can decrypt messages", async function() {
+            const sid = await setupSession(aliceOlmDevice, bobOlmDevice);
+
+            const ciphertext = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sid,
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+
+            const result = await bobOlmDevice.createInboundSession(
+                aliceOlmDevice.deviceCurve25519Key,
+                ciphertext.type,
+                ciphertext.body,
+            );
+            expect(result.payload).toEqual(
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+        });
+    });
+});
@@ -0,0 +1,471 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+import sdk from '../../..';
+import algorithms from '../../../lib/crypto/algorithms';
+import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../MockStorageApi';
+import testUtils from '../../test-utils';
+
+import OlmDevice from '../../../lib/crypto/OlmDevice';
+import Crypto from '../../../lib/crypto';
+
+const Olm = global.Olm;
+
+const MatrixClient = sdk.MatrixClient;
+const MatrixEvent = sdk.MatrixEvent;
+const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
+
+const ROOM_ID = '!ROOM:ID';
+
+const SESSION_ID = 'o+21hSjP+mgEmcfdslPsQdvzWnkdt0Wyo00Kp++R8Kc';
+const ENCRYPTED_EVENT = new MatrixEvent({
+    type: 'm.room.encrypted',
+    room_id: '!ROOM:ID',
+    content: {
+        algorithm: 'm.megolm.v1.aes-sha2',
+        sender_key: 'SENDER_CURVE25519',
+        session_id: SESSION_ID,
+        ciphertext: 'AwgAEjD+VwXZ7PoGPRS/H4kwpAsMp/g+WPvJVtPEKE8fmM9IcT/N'
+            + 'CiwPb8PehecDKP0cjm1XO88k6Bw3D17aGiBHr5iBoP7oSw8CXULXAMTkBl'
+            + 'mkufRQq2+d0Giy1s4/Cg5n13jSVrSb2q7VTSv1ZHAFjUCsLSfR0gxqcQs',
+    },
+    event_id: '$event1',
+    origin_server_ts: 1507753886000,
+});
+
+const KEY_BACKUP_DATA = {
+    first_message_index: 0,
+    forwarded_count: 0,
+    is_verified: false,
+    session_data: {
+        ciphertext: '2z2M7CZ+azAiTHN1oFzZ3smAFFt+LEOYY6h3QO3XXGdw'
+            + '6YpNn/gpHDO6I/rgj1zNd4FoTmzcQgvKdU8kN20u5BWRHxaHTZ'
+            + 'Slne5RxE6vUdREsBgZePglBNyG0AogR/PVdcrv/v18Y6rLM5O9'
+            + 'SELmwbV63uV9Kuu/misMxoqbuqEdG7uujyaEKtjlQsJ5MGPQOy'
+            + 'Syw7XrnesSwF6XWRMxcPGRV0xZr3s9PI350Wve3EncjRgJ9IGF'
+            + 'ru1bcptMqfXgPZkOyGvrphHoFfoK7nY3xMEHUiaTRfRIjq8HNV'
+            + '4o8QY1qmWGnxNBQgOlL8MZlykjg3ULmQ3DtFfQPj/YYGS3jzxv'
+            + 'C+EBjaafmsg+52CTeK3Rswu72PX450BnSZ1i3If4xWAUKvjTpe'
+            + 'Ug5aDLqttOv1pITolTJDw5W/SD+b5rjEKg1CFCHGEGE9wwV3Nf'
+            + 'QHVCQL+dfpd7Or0poy4dqKMAi3g0o3Tg7edIF8d5rREmxaALPy'
+            + 'iie8PHD8mj/5Y0GLqrac4CD6+Mop7eUTzVovprjg',
+        mac: '5lxYBHQU80M',
+        ephemeral: '/Bn0A4UMFwJaDDvh0aEk1XZj3k1IfgCxgFY9P9a0b14',
+    },
+};
+
+const BACKUP_INFO = {
+    algorithm: "m.megolm_backup.v1",
+    version: 1,
+    auth_data: {
+        public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+    },
+};
+
+function makeTestClient(sessionStore, cryptoStore) {
+    const scheduler = [
+        "getQueueForEvent", "queueEvent", "removeEventFromQueue",
+        "setProcessFunction",
+    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+    const store = [
+        "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
+        "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
+        "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
+        "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
+    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+    store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+    store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
+    store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+    return new MatrixClient({
+        baseUrl: "https://my.home.server",
+        idBaseUrl: "https://identity.server",
+        accessToken: "my.access.token",
+        request: function() {}, // NOP
+        store: store,
+        scheduler: scheduler,
+        userId: "@alice:bar",
+        deviceId: "device",
+        sessionStore: sessionStore,
+        cryptoStore: cryptoStore,
+    });
+}
+
+describe("MegolmBackup", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    let olmDevice;
+    let mockOlmLib;
+    let mockCrypto;
+    let mockStorage;
+    let sessionStore;
+    let cryptoStore;
+    let megolmDecryption;
+    beforeEach(async function() {
+        await Olm.init();
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        mockCrypto = testUtils.mock(Crypto, 'Crypto');
+        mockCrypto.backupKey = new Olm.PkEncryption();
+        mockCrypto.backupKey.set_recipient_key(
+            "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+        );
+        mockCrypto.backupInfo = BACKUP_INFO;
+
+        mockStorage = new MockStorageApi();
+        sessionStore = new WebStorageSessionStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore(mockStorage);
+
+        olmDevice = new OlmDevice(sessionStore, cryptoStore);
+
+        // we stub out the olm encryption bits
+        mockOlmLib = {};
+        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.encryptMessageForDevice =
+            expect.createSpy().andReturn(Promise.resolve());
+    });
+
+    describe("backup", function() {
+        let mockBaseApis;
+        let realSetTimeout;
+
+        beforeEach(function() {
+            mockBaseApis = {};
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: mockBaseApis,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            // clobber the setTimeout function to run 100x faster.
+            // ideally we would use lolex, but we have no oportunity
+            // to tick the clock between the first try and the retry.
+            realSetTimeout = global.setTimeout;
+            global.setTimeout = function(f, n) {
+                return realSetTimeout(f, n/100);
+            };
+        });
+
+        afterEach(function() {
+            global.setTimeout = realSetTimeout;
+        });
+
+        it('automatically calls the key back up', function() {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+
+            // construct a fake decrypted key event via the use of a mocked
+            // 'crypto' implementation.
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+            });
+            const decryptedData = {
+                clearEvent: {
+                    type: 'm.room_key',
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        room_id: ROOM_ID,
+                        session_id: groupSession.session_id(),
+                        session_key: groupSession.session_key(),
+                    },
+                },
+                senderCurve25519Key: "SENDER_CURVE25519",
+                claimedEd25519Key: "SENDER_ED25519",
+            };
+
+            mockCrypto.decryptEvent = function() {
+                return Promise.resolve(decryptedData);
+            };
+            mockCrypto.cancelRoomKeyRequest = function() {};
+
+            mockCrypto.backupGroupSession = expect.createSpy();
+
+            return event.attemptDecryption(mockCrypto).then(() => {
+                return megolmDecryption.onRoomKeyEvent(event);
+            }).then(() => {
+                expect(mockCrypto.backupGroupSession).toHaveBeenCalled();
+            });
+        });
+
+        it('sends backups to the server', function() {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+            const ibGroupSession = new Olm.InboundGroupSession();
+            ibGroupSession.create(groupSession.session_key());
+
+            const client = makeTestClient(sessionStore, cryptoStore);
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: client,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            return client.initCrypto()
+                .then(() => {
+                    return cryptoStore.doTxn(
+                        "readwrite",
+                        [cryptoStore.STORE_SESSION],
+                        (txn) => {
+                            cryptoStore.addEndToEndInboundGroupSession(
+                                "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                                groupSession.session_id(),
+                                {
+                                    forwardingCurve25519KeyChain: undefined,
+                                    keysClaimed: {
+                                        ed25519: "SENDER_ED25519",
+                                    },
+                                    room_id: ROOM_ID,
+                                    session: ibGroupSession.pickle(olmDevice._pickleKey),
+                                },
+                                txn);
+                        });
+                })
+                .then(() => {
+                    client.enableKeyBackup({
+                        algorithm: "m.megolm_backup.v1",
+                        version: 1,
+                        auth_data: {
+                            public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                        },
+                    });
+                    let numCalls = 0;
+                    return new Promise((resolve, reject) => {
+                        client._http.authedRequest = function(
+                            callback, method, path, queryParams, data, opts,
+                        ) {
+                            ++numCalls;
+                            expect(numCalls).toBeLessThanOrEqualTo(1);
+                            if (numCalls >= 2) {
+                                // exit out of retry loop if there's something wrong
+                                reject(new Error("authedRequest called too many timmes"));
+                                return Promise.resolve({});
+                            }
+                            expect(method).toBe("PUT");
+                            expect(path).toBe("/room_keys/keys");
+                            expect(queryParams.version).toBe(1);
+                            expect(data.rooms[ROOM_ID].sessions).toExist();
+                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                                groupSession.session_id(),
+                            );
+                            resolve();
+                            return Promise.resolve({});
+                        };
+                        client._crypto.backupGroupSession(
+                            "roomId",
+                            "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                            [],
+                            groupSession.session_id(),
+                            groupSession.session_key(),
+                        );
+                    }).then(() => {
+                        expect(numCalls).toBe(1);
+                    });
+                });
+        });
+
+        it('retries when a backup fails', function() {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+            const ibGroupSession = new Olm.InboundGroupSession();
+            ibGroupSession.create(groupSession.session_key());
+
+            const scheduler = [
+                "getQueueForEvent", "queueEvent", "removeEventFromQueue",
+                "setProcessFunction",
+            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+            const store = [
+                "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
+                "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
+                "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
+                "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
+            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+            store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+            store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
+            store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+            const client = new MatrixClient({
+                baseUrl: "https://my.home.server",
+                idBaseUrl: "https://identity.server",
+                accessToken: "my.access.token",
+                request: function() {}, // NOP
+                store: store,
+                scheduler: scheduler,
+                userId: "@alice:bar",
+                deviceId: "device",
+                sessionStore: sessionStore,
+                cryptoStore: cryptoStore,
+            });
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: client,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            return client.initCrypto()
+                .then(() => {
+                    return cryptoStore.doTxn(
+                        "readwrite",
+                        [cryptoStore.STORE_SESSION],
+                        (txn) => {
+                            cryptoStore.addEndToEndInboundGroupSession(
+                                "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                                groupSession.session_id(),
+                                {
+                                    forwardingCurve25519KeyChain: undefined,
+                                    keysClaimed: {
+                                        ed25519: "SENDER_ED25519",
+                                    },
+                                    room_id: ROOM_ID,
+                                    session: ibGroupSession.pickle(olmDevice._pickleKey),
+                                },
+                                txn);
+                        });
+                })
+                .then(() => {
+                    client.enableKeyBackup({
+                        algorithm: "foobar",
+                        version: 1,
+                        auth_data: {
+                            public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                        },
+                    });
+                    let numCalls = 0;
+                    return new Promise((resolve, reject) => {
+                        client._http.authedRequest = function(
+                            callback, method, path, queryParams, data, opts,
+                        ) {
+                            ++numCalls;
+                            expect(numCalls).toBeLessThanOrEqualTo(2);
+                            if (numCalls >= 3) {
+                                // exit out of retry loop if there's something wrong
+                                reject(new Error("authedRequest called too many timmes"));
+                                return Promise.resolve({});
+                            }
+                            expect(method).toBe("PUT");
+                            expect(path).toBe("/room_keys/keys");
+                            expect(queryParams.version).toBe(1);
+                            expect(data.rooms[ROOM_ID].sessions).toExist();
+                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                                groupSession.session_id(),
+                            );
+                            if (numCalls > 1) {
+                                resolve();
+                                return Promise.resolve({});
+                            } else {
+                                return Promise.reject(
+                                    new Error("this is an expected failure"),
+                                );
+                            }
+                        };
+                        client._crypto.backupGroupSession(
+                            "roomId",
+                            "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI",
+                            [],
+                            groupSession.session_id(),
+                            groupSession.session_key(),
+                        );
+                    }).then(() => {
+                        expect(numCalls).toBe(2);
+                    });
+                });
+        });
+    });
+
+    describe("restore", function() {
+        let client;
+
+        beforeEach(function() {
+            client = makeTestClient(sessionStore, cryptoStore);
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: client,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            return client.initCrypto();
+        });
+
+        afterEach(function() {
+            client.stopClient();
+        });
+
+        it('can restore from backup', function() {
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            return client.restoreKeyBackupWithRecoveryKey(
+                "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                ROOM_ID,
+                SESSION_ID,
+                BACKUP_INFO,
+            ).then(() => {
+                return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
+            }).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
+            });
+        });
+
+        it('can restore backup by room', function() {
+            client._http.authedRequest = function() {
+                return Promise.resolve({
+                    rooms: {
+                        [ROOM_ID]: {
+                            sessions: {
+                                [SESSION_ID]: KEY_BACKUP_DATA,
+                            },
+                        },
+                    },
+                });
+            };
+            return client.restoreKeyBackupWithRecoveryKey(
+                "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                null, null, BACKUP_INFO,
+            ).then(() => {
+                return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
+            }).then((res) => {
+                expect(res.clearEvent.content).toEqual('testytest');
+            });
+        });
+    });
+});
@@ -0,0 +1,146 @@
+/*
+Copyright 2018-2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run device verification tests: libolm not available");
+}
+
+import expect from 'expect';
+
+import DeviceInfo from '../../../../lib/crypto/deviceinfo';
+
+import {ShowQRCode, ScanQRCode} from '../../../../lib/crypto/verification/QRCode';
+
+const Olm = global.Olm;
+
+describe("QR code verification", function() {
+    if (!global.Olm) {
+        console.warn('Not running device verification tests: libolm not present');
+        return;
+    }
+
+    beforeEach(async function() {
+        await Olm.init();
+    });
+
+    describe("showing", function() {
+        it("should emit an event to show a QR code", async function() {
+            const qrCode = new ShowQRCode({
+                getUserId: () => "@alice:example.com",
+                deviceId: "ABCDEFG",
+                getDeviceEd25519Key: function() {
+                    return "device+ed25519+key";
+                },
+            });
+            const spy = expect.createSpy().andCall((e) => {
+                qrCode.done();
+            });
+            qrCode.on("show_qr_code", spy);
+            await qrCode.verify();
+            expect(spy).toHaveBeenCalledWith({
+                url: "https://matrix.to/#/@alice:example.com?device=ABCDEFG"
+                    + "&action=verify&key_ed25519%3AABCDEFG=device%2Bed25519%2Bkey",
+            });
+        });
+    });
+
+    describe("scanning", function() {
+        const QR_CODE_URL = "https://matrix.to/#/@alice:example.com?device=ABCDEFG"
+              + "&action=verify&key_ed25519%3AABCDEFG=device%2Bed25519%2Bkey";
+        it("should verify when a QR code is sent", async function() {
+            const device = DeviceInfo.fromStorage(
+                {
+                    algorithms: [],
+                    keys: {
+                        "curve25519:ABCDEFG": "device+curve25519+key",
+                        "ed25519:ABCDEFG": "device+ed25519+key",
+                    },
+                    verified: false,
+                    known: false,
+                    unsigned: {},
+                },
+                "ABCDEFG",
+            );
+            const client = {
+                getStoredDevice: expect.createSpy().andReturn(device),
+                setDeviceVerified: expect.createSpy(),
+            };
+            const qrCode = new ScanQRCode(client);
+            qrCode.on("confirm_user_id", ({userId, confirm}) => {
+                if (userId === "@alice:example.com") {
+                    confirm();
+                } else {
+                    qrCode.cancel(new Error("Incorrect user"));
+                }
+            });
+            qrCode.on("scan", ({done}) => {
+                done(QR_CODE_URL);
+            });
+            await qrCode.verify();
+            expect(client.getStoredDevice)
+                .toHaveBeenCalledWith("@alice:example.com", "ABCDEFG");
+            expect(client.setDeviceVerified)
+                .toHaveBeenCalledWith("@alice:example.com", "ABCDEFG");
+        });
+
+        it("should error when the user ID doesn't match", async function() {
+            const client = {
+                getStoredDevice: expect.createSpy(),
+                setDeviceVerified: expect.createSpy(),
+            };
+            const qrCode = new ScanQRCode(client, "@bob:example.com", "ABCDEFG");
+            qrCode.on("scan", ({done}) => {
+                done(QR_CODE_URL);
+            });
+            const spy = expect.createSpy();
+            await qrCode.verify().catch(spy);
+            expect(spy).toHaveBeenCalled();
+            expect(client.getStoredDevice).toNotHaveBeenCalled();
+            expect(client.setDeviceVerified).toNotHaveBeenCalled();
+        });
+
+        it("should error if the key doesn't match", async function() {
+            const device = DeviceInfo.fromStorage(
+                {
+                    algorithms: [],
+                    keys: {
+                        "curve25519:ABCDEFG": "device+curve25519+key",
+                        "ed25519:ABCDEFG": "a+different+device+ed25519+key",
+                    },
+                    verified: false,
+                    known: false,
+                    unsigned: {},
+                },
+                "ABCDEFG",
+            );
+            const client = {
+                getStoredDevice: expect.createSpy().andReturn(device),
+                setDeviceVerified: expect.createSpy(),
+            };
+            const qrCode = new ScanQRCode(client, "@alice:example.com", "ABCDEFG");
+            qrCode.on("scan", ({done}) => {
+                done(QR_CODE_URL);
+            });
+            const spy = expect.createSpy();
+            await qrCode.verify().catch(spy);
+            expect(spy).toHaveBeenCalled();
+            expect(client.getStoredDevice).toHaveBeenCalled();
+            expect(client.setDeviceVerified).toNotHaveBeenCalled();
+        });
+    });
+});
@@ -0,0 +1,75 @@
+/*
+Copyright 2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run device verification tests: libolm not available");
+}
+
+import expect from 'expect';
+
+import {verificationMethods} from '../../../../lib/crypto';
+
+import SAS from '../../../../lib/crypto/verification/SAS';
+
+const Olm = global.Olm;
+
+import {makeTestClients} from './util';
+
+describe("verification request", function() {
+    if (!global.Olm) {
+        console.warn('Not running device verification unit tests: libolm not present');
+        return;
+    }
+
+    beforeEach(async function() {
+        await Olm.init();
+    });
+
+    it("should request and accept a verification", async function() {
+        const [alice, bob] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@bob:example.com", deviceId: "Dynabook"},
+            ],
+            {
+                verificationMethods: [verificationMethods.SAS],
+            },
+        );
+        alice._crypto._deviceList.getRawStoredDevicesForUser = function() {
+            return {
+                Dynabook: {
+                    keys: {
+                        "ed25519:Dynabook": "bob+base64+ed25519+key",
+                    },
+                },
+            };
+        };
+        alice.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.on("crypto.verification.request", (request) => {
+            const bobVerifier = request.beginKeyVerification(verificationMethods.SAS);
+            bobVerifier.verify();
+        });
+        const aliceVerifier = await alice.requestVerification("@bob:example.com");
+        expect(aliceVerifier).toBeAn(SAS);
+    });
+});
@@ -0,0 +1,207 @@
+/*
+Copyright 2018-2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run device verification tests: libolm not available");
+}
+
+import expect from 'expect';
+
+import sdk from '../../../..';
+
+import {verificationMethods} from '../../../../lib/crypto';
+import DeviceInfo from '../../../../lib/crypto/deviceinfo';
+
+import SAS from '../../../../lib/crypto/verification/SAS';
+
+const Olm = global.Olm;
+
+const MatrixEvent = sdk.MatrixEvent;
+
+import {makeTestClients} from './util';
+
+describe("SAS verification", function() {
+    if (!global.Olm) {
+        console.warn('Not running device verification unit tests: libolm not present');
+        return;
+    }
+
+    beforeEach(async function() {
+        await Olm.init();
+    });
+
+    it("should error on an unexpected event", async function() {
+        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        sas.handleEvent(new MatrixEvent({
+            sender: "@alice:example.com",
+            type: "es.inquisition",
+            content: {},
+        }));
+        const spy = expect.createSpy();
+        await sas.verify()
+            .catch(spy);
+        expect(spy).toHaveBeenCalled();
+    });
+
+    it("should verify a key", async function() {
+        const [alice, bob] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@bob:example.com", deviceId: "Dynabook"},
+            ],
+            {
+                verificationMethods: [verificationMethods.SAS],
+            },
+        );
+
+        alice.setDeviceVerified = expect.createSpy();
+        alice.getDeviceEd25519Key = () => {
+            return "alice+base64+ed25519+key";
+        };
+        alice.getStoredDevice = () => {
+            return DeviceInfo.fromStorage(
+                {
+                    keys: {
+                        "ed25519:Dynabook": "bob+base64+ed25519+key",
+                    },
+                },
+                "Dynabook",
+            );
+        };
+        alice.downloadKeys = () => {
+            return Promise.resolve();
+        };
+
+        bob.setDeviceVerified = expect.createSpy();
+        bob.getStoredDevice = () => {
+            return DeviceInfo.fromStorage(
+                {
+                    keys: {
+                        "ed25519:Osborne2": "alice+base64+ed25519+key",
+                    },
+                },
+                "Osborne2",
+            );
+        };
+        bob.getDeviceEd25519Key = () => {
+            return "bob+base64+ed25519+key";
+        };
+        bob.downloadKeys = () => {
+            return Promise.resolve();
+        };
+
+        let aliceSasEvent;
+        let bobSasEvent;
+
+        const bobPromise = new Promise((resolve, reject) => {
+            bob.on("crypto.verification.start", (verifier) => {
+                verifier.on("show_sas", (e) => {
+                    if (!e.sas.emoji || !e.sas.decimal) {
+                        e.cancel();
+                    } else if (!aliceSasEvent) {
+                        bobSasEvent = e;
+                    } else {
+                        try {
+                            expect(e.sas).toEqual(aliceSasEvent.sas);
+                            e.confirm();
+                            aliceSasEvent.confirm();
+                        } catch (error) {
+                            e.mismatch();
+                            aliceSasEvent.mismatch();
+                        }
+                    }
+                });
+                resolve(verifier);
+            });
+        });
+
+        const aliceVerifier = alice.beginKeyVerification(
+            verificationMethods.SAS, bob.getUserId(), bob.deviceId,
+        );
+        aliceVerifier.on("show_sas", (e) => {
+            if (!e.sas.emoji || !e.sas.decimal) {
+                e.cancel();
+            } else if (!bobSasEvent) {
+                aliceSasEvent = e;
+            } else {
+                try {
+                    expect(e.sas).toEqual(bobSasEvent.sas);
+                    e.confirm();
+                    bobSasEvent.confirm();
+                } catch (error) {
+                    e.mismatch();
+                    bobSasEvent.mismatch();
+                }
+            }
+        });
+        await Promise.all([
+            aliceVerifier.verify(),
+            bobPromise.then((verifier) => verifier.verify()),
+        ]);
+        expect(alice.setDeviceVerified)
+            .toHaveBeenCalledWith(bob.getUserId(), bob.deviceId);
+        expect(bob.setDeviceVerified)
+            .toHaveBeenCalledWith(alice.getUserId(), alice.deviceId);
+    });
+
+    it("should send a cancellation message on error", async function() {
+        const [alice, bob] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@bob:example.com", deviceId: "Dynabook"},
+            ],
+            {
+                verificationMethods: [verificationMethods.SAS],
+            },
+        );
+        alice.setDeviceVerified = expect.createSpy();
+        alice.downloadKeys = () => {
+            return Promise.resolve();
+        };
+        bob.setDeviceVerified = expect.createSpy();
+        bob.downloadKeys = () => {
+            return Promise.resolve();
+        };
+
+        const bobPromise = new Promise((resolve, reject) => {
+            bob.on("crypto.verification.start", (verifier) => {
+                verifier.on("show_sas", (e) => {
+                    e.mismatch();
+                });
+                resolve(verifier);
+            });
+        });
+
+        const aliceVerifier = alice.beginKeyVerification(
+            verificationMethods.SAS, bob.getUserId(), bob.deviceId,
+        );
+
+        const aliceSpy = expect.createSpy();
+        const bobSpy = expect.createSpy();
+        await Promise.all([
+            aliceVerifier.verify().catch(aliceSpy),
+            bobPromise.then((verifier) => verifier.verify()).catch(bobSpy),
+        ]);
+        expect(aliceSpy).toHaveBeenCalled();
+        expect(bobSpy).toHaveBeenCalled();
+        expect(alice.setDeviceVerified)
+            .toNotHaveBeenCalled();
+        expect(bob.setDeviceVerified)
+            .toNotHaveBeenCalled();
+    });
+});
@@ -0,0 +1,63 @@
+/*
+Copyright 2019 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import TestClient from '../../../TestClient';
+
+import sdk from '../../../..';
+const MatrixEvent = sdk.MatrixEvent;
+
+export async function makeTestClients(userInfos, options) {
+    const clients = [];
+    const clientMap = {};
+    const sendToDevice = function(type, map) {
+        // console.log(this.getUserId(), "sends", type, map);
+        for (const [userId, devMap] of Object.entries(map)) {
+            if (userId in clientMap) {
+                for (const [deviceId, msg] of Object.entries(devMap)) {
+                    if (deviceId in clientMap[userId]) {
+                        const event = new MatrixEvent({
+                            sender: this.getUserId(), // eslint-disable-line no-invalid-this
+                            type: type,
+                            content: msg,
+                        });
+                        setTimeout(
+                            () => clientMap[userId][deviceId]
+                                .emit("toDeviceEvent", event),
+                            0,
+                        );
+                    }
+                }
+            }
+        }
+    };
+
+    for (const userInfo of userInfos) {
+        const client = (new TestClient(
+            userInfo.userId, userInfo.deviceId, undefined, undefined,
+            options,
+        )).client;
+        if (!(userInfo.userId in clientMap)) {
+            clientMap[userInfo.userId] = {};
+        }
+        clientMap[userInfo.userId][userInfo.deviceId] = client;
+        client.sendToDevice = sendToDevice;
+        clients.push(client);
+    }
+
+    await Promise.all(clients.map((client) => client.initCrypto()));
+
+    return clients;
+}
@@ -0,0 +1,82 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import sdk from '../..';
+const MatrixEvent = sdk.MatrixEvent;
+
+import testUtils from '../test-utils';
+
+import expect from 'expect';
+import Promise from 'bluebird';
+
+describe("MatrixEvent", () => {
+    beforeEach(function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+    });
+
+    describe(".attemptDecryption", () => {
+        let encryptedEvent;
+
+        beforeEach(() => {
+            encryptedEvent = new MatrixEvent({
+                id: 'test_encrypted_event',
+                type: 'm.room.encrypted',
+                content: {
+                    ciphertext: 'secrets',
+                },
+            });
+        });
+
+        it('should retry decryption if a retry is queued', () => {
+            let callCount = 0;
+
+            let prom2;
+
+            const crypto = {
+                decryptEvent: function() {
+                    ++callCount;
+                    console.log(`decrypt: ${callCount}`);
+                    if (callCount == 1) {
+                        // schedule a second decryption attempt while
+                        // the first one is still running.
+                        prom2 = encryptedEvent.attemptDecryption(crypto);
+
+                        const error = new Error("nope");
+                        error.name = 'DecryptionError';
+                        return Promise.reject(error);
+                    } else {
+                        expect(prom2.isFulfilled()).toBe(
+                            false, 'second attemptDecryption resolved too soon');
+
+                        return Promise.resolve({
+                            clearEvent: {
+                                type: 'm.room.message',
+                            },
+                        });
+                    }
+                },
+            };
+
+            return encryptedEvent.attemptDecryption(crypto).then(() => {
+                expect(callCount).toEqual(2);
+                expect(encryptedEvent.getType()).toEqual('m.room.message');
+
+                // make sure the second attemptDecryption resolves
+                return prom2;
+            });
+        });
+    });
+});
@@ -16,7 +16,7 @@ limitations under the License.
 "use strict";

 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const utils = require("../test-utils");

@@ -81,14 +81,14 @@ describe("InteractiveAuth", function() {
                type: "logintype",
                foo: "bar",
            });
-            return q(requestRes);
+            return Promise.resolve(requestRes);
        });

        ia.attemptAuth().then(function(res) {
            expect(res).toBe(requestRes);
            expect(doRequest.calls.length).toEqual(1);
            expect(stateUpdated.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
    });

    it("should make a request if no authdata is provided", function(done) {
@@ -138,7 +138,7 @@ describe("InteractiveAuth", function() {
                    type: "logintype",
                    foo: "bar",
                });
-                return q(requestRes);
+                return Promise.resolve(requestRes);
            });

            ia.submitAuthDict({
@@ -151,6 +151,6 @@ describe("InteractiveAuth", function() {
            expect(res).toBe(requestRes);
            expect(doRequest.calls.length).toEqual(2);
            expect(stateUpdated.calls.length).toEqual(1);
-        }).catch(utils.failTest).done(done);
+        }).nodeify(done);
    });
 });
@@ -1,6 +1,6 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const MatrixClient = sdk.MatrixClient;
 const utils = require("../test-utils");
@@ -62,7 +62,7 @@ describe("MatrixClient", function() {
    let pendingLookup = null;
    function httpReq(cb, method, path, qp, data, prefix) {
        if (path === KEEP_ALIVE_PATH && acceptKeepalives) {
-            return q();
+            return Promise.resolve();
        }
        const next = httpLookups.shift();
        const logLine = (
@@ -84,7 +84,7 @@ describe("MatrixClient", function() {
                );
            }
            pendingLookup = {
-                promise: q.defer().promise,
+                promise: Promise.defer().promise,
                method: method,
                path: path,
            };
@@ -109,7 +109,7 @@ describe("MatrixClient", function() {
            }

            if (next.error) {
-                return q.reject({
+                return Promise.reject({
                    errcode: next.error.errcode,
                    httpStatus: next.error.httpStatus,
                    name: next.error.errcode,
@@ -117,10 +117,10 @@ describe("MatrixClient", function() {
                    data: next.error,
                });
            }
-            return q(next.data);
+            return Promise.resolve(next.data);
        }
        expect(true).toBe(false, "Expected different request. " + logLine);
-        return q.defer().promise;
+        return Promise.defer().promise;
    }

    beforeEach(function() {
@@ -132,12 +132,16 @@ describe("MatrixClient", function() {
        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
        store = [
            "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "save", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
            "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
            "getSyncAccumulator", "startup", "deleteAllData",
        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        store.getSavedSync = expect.createSpy().andReturn(q(null));
-        store.setSyncData = expect.createSpy().andReturn(q(null));
+        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
+        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.storeClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.isNewlyCreated = expect.createSpy().andReturn(Promise.resolve(true));
        client = new MatrixClient({
            baseUrl: "https://my.home.server",
            idBaseUrl: identityServerUrl,
@@ -174,14 +178,14 @@ describe("MatrixClient", function() {
        // a DIFFERENT test (pollution between tests!) - we return unresolved
        // promises to stop the client from continuing to run.
        client._http.authedRequest.andCall(function() {
-            return q.defer().promise;
+            return Promise.defer().promise;
        });
        client._http.authedRequestWithPrefix.andCall(function() {
-            return q.defer().promise;
+            return Promise.defer().promise;
        });
    });

-    it("should not POST /filter if a matching filter already exists", function(done) {
+    it("should not POST /filter if a matching filter already exists", async function() {
        httpLookups = [];
        httpLookups.push(PUSH_RULES_RESPONSE);
        httpLookups.push(SYNC_RESPONSE);
@@ -190,15 +194,19 @@ describe("MatrixClient", function() {
        const filter = new sdk.Filter(0, filterId);
        filter.setDefinition({"room": {"timeline": {"limit": 8}}});
        store.getFilter.andReturn(filter);
-        client.startClient();
-
-        client.on("sync", function syncListener(state) {
-            if (state === "SYNCING") {
-                expect(httpLookups.length).toEqual(0);
-                client.removeListener("sync", syncListener);
-                done();
-            }
+        const syncPromise = new Promise((resolve, reject) => {
+            client.on("sync", function syncListener(state) {
+                if (state === "SYNCING") {
+                    expect(httpLookups.length).toEqual(0);
+                    client.removeListener("sync", syncListener);
+                    resolve();
+                } else if (state === "ERROR") {
+                    reject(new Error("sync error"));
+                }
+            });
        });
+        await client.startClient();
+        await syncPromise;
    });

    describe("getSyncState", function() {
@@ -206,15 +214,18 @@ describe("MatrixClient", function() {
            expect(client.getSyncState()).toBe(null);
        });

-        it("should return the same sync state as emitted sync events", function(done) {
-            client.on("sync", function syncListener(state) {
-                expect(state).toEqual(client.getSyncState());
-                if (state === "SYNCING") {
-                    client.removeListener("sync", syncListener);
-                    done();
-                }
+        it("should return the same sync state as emitted sync events", async function() {
+            const syncingPromise = new Promise((resolve) => {
+                client.on("sync", function syncListener(state) {
+                    expect(state).toEqual(client.getSyncState());
+                    if (state === "SYNCING") {
+                        client.removeListener("sync", syncListener);
+                        resolve();
+                    }
+                });
            });
-            client.startClient();
+            await client.startClient();
+            await syncingPromise;
        });
    });

@@ -257,8 +268,8 @@ describe("MatrixClient", function() {
    });

    describe("retryImmediately", function() {
-        it("should return false if there is no request waiting", function() {
-            client.startClient();
+        it("should return false if there is no request waiting", async function() {
+            await client.startClient();
            expect(client.retryImmediately()).toBe(false);
        });

@@ -379,7 +390,7 @@ describe("MatrixClient", function() {
            client.startClient();
        });

-        it("should transition ERROR -> PREPARED after /sync if prev failed",
+        it("should transition ERROR -> CATCHUP after /sync if prev failed",
        function(done) {
            const expectedStates = [];
            acceptKeepalives = false;
@@ -402,7 +413,7 @@ describe("MatrixClient", function() {

            expectedStates.push(["RECONNECTING", null]);
            expectedStates.push(["ERROR", "RECONNECTING"]);
-            expectedStates.push(["PREPARED", "ERROR"]);
+            expectedStates.push(["CATCHUP", "ERROR"]);
            client.on("sync", syncChecker(expectedStates, done));
            client.startClient();
        });
@@ -24,6 +24,9 @@ describe('NotificationService', function() {
                            name: testDisplayName,
                        };
                    },
+                    getJoinedMemberCount: function() {
+                        return 0;
+                    },
                    members: {},
                },
            };
@@ -192,6 +192,15 @@ describe("RoomMember", function() {
        });
    });

+    describe("isOutOfBand", function() {
+        it("should be set by markOutOfBand", function() {
+            const member = new RoomMember();
+            expect(member.isOutOfBand()).toEqual(false);
+            member.markOutOfBand();
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+    });
+
    describe("setMembershipEvent", function() {
        const joinEvent = utils.mkMembership({
            event: true,
@@ -276,5 +285,52 @@ describe("RoomMember", function() {
            member.setMembershipEvent(joinEvent); // no-op
            expect(emitCount).toEqual(1);
        });
+
+        it("should set 'name' to user_id if it is just whitespace", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: " \u200b ",
+            });
+
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent);
+            expect(member.name).toEqual(userA); // it should fallback because all whitespace
+        });
+
+        it("should disambiguate users on a fuzzy displayname match", function() {
+            const joinEvent = utils.mkMembership({
+                event: true,
+                mship: "join",
+                user: userA,
+                room: roomId,
+                name: "Alíce\u200b", // note diacritic and zero width char
+            });
+
+            const roomState = {
+                getStateEvents: function(type) {
+                    if (type !== "m.room.member") {
+                        return [];
+                    }
+                    return [
+                        utils.mkMembership({
+                            event: true, mship: "join", room: roomId,
+                            user: userC, name: "Alice",
+                        }),
+                        joinEvent,
+                    ];
+                },
+                getUserIdsWithDisplayName: function(displayName) {
+                    return [userA, userC];
+                },
+            };
+            expect(member.name).toEqual(userA); // default = user_id
+            member.setMembershipEvent(joinEvent, roomState);
+            expect(member.name).toNotEqual("Alíce"); // it should disambig.
+            // user_id should be there somewhere
+            expect(member.name.indexOf(userA)).toNotEqual(-1);
+        });
    });
 });
@@ -11,6 +11,9 @@ describe("RoomState", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
    const userB = "@bob:bar";
+    const userC = "@cleo:bar";
+    const userLazy = "@lazy:bar";
+
    let state;

    beforeEach(function() {
@@ -78,8 +81,8 @@ describe("RoomState", function() {
    });

    describe("getSentinelMember", function() {
-        it("should return null if there is no member", function() {
-            expect(state.getSentinelMember("@no-one:here")).toEqual(null);
+        it("should return a member with the user id as name", function() {
+            expect(state.getSentinelMember("@no-one:here").name).toEqual("@no-one:here");
        });

        it("should return a member which doesn't change when the state is updated",
@@ -162,6 +165,7 @@ describe("RoomState", function() {
            ];
            let emitCount = 0;
            state.on("RoomState.newMember", function(ev, st, mem) {
+                expect(state.getMember(mem.userId)).toEqual(mem);
                expect(mem.userId).toEqual(memberEvents[emitCount].getSender());
                expect(mem.membership).toBeFalsy();  // not defined yet
                emitCount += 1;
@@ -222,7 +226,6 @@ describe("RoomState", function() {

        it("should call setPowerLevelEvent on a new RoomMember if power levels exist",
        function() {
-            const userC = "@cleo:bar";
            const memberEvent = utils.mkMembership({
                mship: "join", user: userC, room: roomId, event: true,
            });
@@ -262,6 +265,114 @@ describe("RoomState", function() {
        });
    });

+    describe("setOutOfBandMembers", function() {
+        it("should add a new member", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const member = state.getMember(userLazy);
+            expect(member.userId).toEqual(userLazy);
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+
+        it("should have no effect when not in correct status", function() {
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should emit newMember when adding a member", function() {
+            const userLazy = "@oob:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.newMember', (_, __, member) => {
+                expect(member.userId).toEqual(userLazy);
+                eventReceived = true;
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+
+        it("should never overwrite existing members", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const memberA = state.getMember(userA);
+            expect(memberA.events.member.getId()).toNotEqual(oobMemberEvent.getId());
+            expect(memberA.isOutOfBand()).toEqual(false);
+        });
+
+        it("should emit members when updating a member", function() {
+            const doesntExistYetUserId = "@doesntexistyet:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: doesntExistYetUserId, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.members', (_, __, member) => {
+                expect(member.userId).toEqual(doesntExistYetUserId);
+                eventReceived = true;
+            });
+
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+    });
+
+    describe("clone", function() {
+        it("should contain same information as original", function() {
+            // include OOB members in copy
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            const copy = state.clone();
+            // check individual members
+            [userA, userB, userLazy].forEach((userId) => {
+                const member = state.getMember(userId);
+                const memberCopy = copy.getMember(userId);
+                expect(member.name).toEqual(memberCopy.name);
+                expect(member.isOutOfBand()).toEqual(memberCopy.isOutOfBand());
+            });
+            // check member keys
+            expect(Object.keys(state.members)).toEqual(Object.keys(copy.members));
+            // check join count
+            expect(state.getJoinedMemberCount()).toEqual(copy.getJoinedMemberCount());
+        });
+
+        it("should mark old copy as not waiting for out of band anymore", function() {
+            state.markOutOfBandMembersStarted();
+            const copy = state.clone();
+            copy.setOutOfBandMembers([utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            })]);
+            // should have no effect as it should be marked in status finished just like copy
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should return copy independent of original", function() {
+            const copy = state.clone();
+            copy.setStateEvents([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+
+            expect(state.getMember(userLazy)).toBeFalsy();
+            expect(state.getJoinedMemberCount()).toEqual(2);
+            expect(copy.getJoinedMemberCount()).toEqual(3);
+        });
+    });
+
    describe("setTypingEvent", function() {
        it("should call setTypingEvent on each RoomMember", function() {
            const typingEvent = utils.mkEvent({
@@ -284,13 +395,6 @@ describe("RoomState", function() {
    });

    describe("maySendStateEvent", function() {
-        it("should say non-joined members may not send state",
-        function() {
-            expect(state.maySendStateEvent(
-                'm.room.name', "@nobody:nowhere",
-            )).toEqual(false);
-        });
-
        it("should say any member may send state with no power level event",
        function() {
            expect(state.maySendStateEvent('m.room.name', userA)).toEqual(true);
@@ -366,15 +470,117 @@ describe("RoomState", function() {
        });
    });

-    describe("maySendEvent", function() {
-        it("should say non-joined members may not send events",
-        function() {
-            expect(state.maySendEvent(
-                'm.room.message', "@nobody:nowhere",
-            )).toEqual(false);
-            expect(state.maySendMessage("@nobody:nowhere")).toEqual(false);
+    describe("getJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
        });

+        it("should update after adding joined member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("getInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should update after adding invited member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("setJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setJoinedMemberCount(100);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            state.setJoinedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("setInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setInvitedMemberCount(100);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            state.setInvitedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("maySendEvent", function() {
        it("should say any member may send events with no power level event",
        function() {
            expect(state.maySendEvent('m.room.message', userA)).toEqual(true);
@@ -67,13 +67,14 @@ describe("Room", function() {

    describe("getMember", function() {
        beforeEach(function() {
-            // clobber members property with test data
-            room.currentState.members = {
-                "@alice:bar": {
-                    userId: userA,
-                    roomId: roomId,
-                },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": {
+                        userId: userA,
+                        roomId: roomId,
+                    },
+                }[userId];
+            });
        });

        it("should return null if the member isn't in current state", function() {
@@ -383,28 +384,31 @@ describe("Room", function() {
    });

    const resetTimelineTests = function(timelineSupport) {
-        const events = [
-            utils.mkMessage({
-                room: roomId, user: userA, msg: "A message", event: true,
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "New Room Name" },
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "Another New Name" },
-            }),
-        ];
+        let events = null;

        beforeEach(function() {
-            room = new Room(roomId, {timelineSupport: timelineSupport});
+            room = new Room(roomId, null, null, {timelineSupport: timelineSupport});
+            // set events each time to avoid resusing Event objects (which
+            // doesn't work because they get frozen)
+            events = [
+                utils.mkMessage({
+                    room: roomId, user: userA, msg: "A message", event: true,
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "New Room Name" },
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "Another New Name" },
+                }),
+            ];
        });

        it("should copy state from previous timeline", function() {
            room.addLiveEvents([events[0], events[1]]);
            expect(room.getLiveTimeline().getEvents().length).toEqual(2);
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');

            room.addLiveEvents([events[2]]);
            const oldState = room.getLiveTimeline().getState(EventTimeline.BACKWARDS);
@@ -417,7 +421,7 @@ describe("Room", function() {
        it("should reset the legacy timeline fields", function() {
            room.addLiveEvents([events[0], events[1]]);
            expect(room.timeline.length).toEqual(2);
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');

            room.addLiveEvents([events[2]]);
            const newLiveTimeline = room.getLiveTimeline();
@@ -451,7 +455,7 @@ describe("Room", function() {
            room.addLiveEvents([events[0]]);
            expect(room.timeline.length).toEqual(1);
            const firstLiveTimeline = room.getLiveTimeline();
-            room.resetLiveTimeline();
+            room.resetLiveTimeline('sometoken', 'someothertoken');

            const tl = room.getTimelineForEvent(events[0].getId());
            expect(tl).toBe(timelineSupport ? firstLiveTimeline : null);
@@ -465,7 +469,7 @@ describe("Room", function() {

    describe("compareEventOrdering", function() {
        beforeEach(function() {
-            room = new Room(roomId, {timelineSupport: true});
+            room = new Room(roomId, null, null, {timelineSupport: true});
        });

        const events = [
@@ -567,72 +571,75 @@ describe("Room", function() {
    describe("hasMembershipState", function() {
        it("should return true for a matching userId and membership",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-                "@bob:bar": { userId: "@bob:bar", membership: "invite" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                    "@bob:bar": { userId: "@bob:bar", membership: "invite" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "invite")).toBe(true);
        });

        it("should return false if match membership but no match userId",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "join")).toBe(false);
        });

        it("should return false if match userId but no match membership",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@alice:bar", "ban")).toBe(false);
        });

        it("should return false if no match membership or userId",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "invite")).toBe(false);
        });

        it("should return false if no members exist",
        function() {
-            room.currentState.members = {};
            expect(room.hasMembershipState("@foo:bar", "join")).toBe(false);
        });
    });

    describe("recalculate", function() {
-        let stateLookup = {
-            // event.type + "$" event.state_key : MatrixEvent
-        };
-
        const setJoinRule = function(rule) {
-            stateLookup["m.room.join_rules$"] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.join_rules", room: roomId, user: userA, content: {
                    join_rule: rule,
                }, event: true,
-            });
+            })]);
        };
        const setAliases = function(aliases, stateKey) {
            if (!stateKey) {
                stateKey = "flibble";
            }
-            stateLookup["m.room.aliases$" + stateKey] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
                    aliases: aliases,
                }, event: true,
-            });
+            })]);
        };
        const setRoomName = function(name) {
-            stateLookup["m.room.name$"] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.name", room: roomId, user: userA, content: {
                    name: name,
                }, event: true,
-            });
+            })]);
        };
        const addMember = function(userId, state, opts) {
            if (!state) {
@@ -644,56 +651,14 @@ describe("Room", function() {
            opts.user = opts.user || userId;
            opts.skey = userId;
            opts.event = true;
-            stateLookup["m.room.member$" + userId] = utils.mkMembership(opts);
+            const event = utils.mkMembership(opts);
+            room.addLiveEvents([event]);
+            return event;
        };

        beforeEach(function() {
-            stateLookup = {};
-            room.currentState.getStateEvents.andCall(function(type, key) {
-                if (key === undefined) {
-                    const prefix = type + "$";
-                    const list = [];
-                    for (const stateBlob in stateLookup) {
-                        if (!stateLookup.hasOwnProperty(stateBlob)) {
-                            continue;
-                        }
-                        if (stateBlob.indexOf(prefix) === 0) {
-                            list.push(stateLookup[stateBlob]);
-                        }
-                    }
-                    return list;
-                } else {
-                    return stateLookup[type + "$" + key];
-                }
-            });
-            room.currentState.getMembers.andCall(function() {
-                const memberEvents = room.currentState.getStateEvents("m.room.member");
-                const members = [];
-                for (let i = 0; i < memberEvents.length; i++) {
-                    members.push({
-                        name: memberEvents[i].event.content &&
-                                memberEvents[i].event.content.displayname ?
-                                memberEvents[i].event.content.displayname :
-                                memberEvents[i].getStateKey(),
-                        userId: memberEvents[i].getStateKey(),
-                        events: { member: memberEvents[i] },
-                    });
-                }
-                return members;
-            });
-            room.currentState.getMember.andCall(function(userId) {
-                const memberEvent = room.currentState.getStateEvents(
-                    "m.room.member", userId,
-                );
-                return {
-                    name: memberEvent.event.content &&
-                            memberEvent.event.content.displayname ?
-                            memberEvent.event.content.displayname :
-                            memberEvent.getStateKey(),
-                    userId: memberEvent.getStateKey(),
-                    events: { member: memberEvent },
-                };
-            });
+            // no mocking
+            room = new Room(roomId, null, userA);
        });

        describe("Room.recalculate => Stripped State Events", function() {
@@ -701,8 +666,8 @@ describe("Room", function() {
            "room is an invite room", function() {
                const roomName = "flibble";

-                addMember(userA, "invite");
-                stateLookup["m.room.member$" + userA].event.invite_room_state = [
+                const event = addMember(userA, "invite");
+                event.event.invite_room_state = [
                    {
                        type: "m.room.name",
                        state_key: "",
@@ -712,30 +677,108 @@ describe("Room", function() {
                    },
                ];

-                room.recalculate(userA);
-                expect(room.currentState.setStateEvents).toHaveBeenCalled();
-                // first call, first arg (which is an array), first element in array
-                const fakeEvent = room.currentState.setStateEvents.calls[0].
-                      arguments[0][0];
-                expect(fakeEvent.getContent()).toEqual({
-                    name: roomName,
-                });
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
            });

            it("should not clobber state events if it isn't an invite room", function() {
-                addMember(userA, "join");
-                stateLookup["m.room.member$" + userA].event.invite_room_state = [
+                const event = addMember(userA, "join");
+                const roomName = "flibble";
+                setRoomName(roomName);
+                const roomNameToIgnore = "ignoreme";
+                event.event.invite_room_state = [
                    {
                        type: "m.room.name",
                        state_key: "",
                        content: {
-                            name: "flibble",
+                            name: roomNameToIgnore,
                        },
                    },
                ];

-                room.recalculate(userA);
-                expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
+            });
+        });
+
+        describe("Room.recalculate => Room Name using room summary", function() {
+            it("should use room heroes if available", function() {
+                addMember(userA, "invite");
+                addMember(userB);
+                addMember(userC);
+                addMember(userD);
+                room.setSummary({
+                    "m.heroes": [userB, userC, userD],
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(`${userB} and 2 others`);
+            });
+
+            it("missing hero member state reverts to mxid", function() {
+                room.setSummary({
+                    "m.heroes": [userB],
+                    "m.joined_member_count": 2,
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(userB);
+            });
+
+            it("uses hero name from state", function() {
+                const name = "Mr B";
+                addMember(userA, "invite");
+                addMember(userB, "join", {name});
+                room.setSummary({
+                    "m.heroes": [userB],
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(name);
+            });
+
+            it("uses counts from summary", function() {
+                const name = "Mr B";
+                addMember(userB, "join", {name});
+                room.setSummary({
+                    "m.heroes": [userB],
+                    "m.joined_member_count": 50,
+                    "m.invited_member_count": 50,
+                });
+                room.recalculate();
+                expect(room.name).toEqual(`${name} and 98 others`);
+            });
+
+            it("relies on heroes in case of absent counts", function() {
+                const nameB = "Mr Bean";
+                const nameC = "Mel C";
+                addMember(userB, "join", {name: nameB});
+                addMember(userC, "join", {name: nameC});
+                room.setSummary({
+                    "m.heroes": [userB, userC],
+                });
+                room.recalculate();
+                expect(room.name).toEqual(`${nameB} and ${nameC}`);
+            });
+
+            it("uses only heroes", function() {
+                const nameB = "Mr Bean";
+                addMember(userB, "join", {name: nameB});
+                addMember(userC, "join");
+                room.setSummary({
+                    "m.heroes": [userB],
+                });
+                room.recalculate();
+                expect(room.name).toEqual(nameB);
+            });
+
+            it("reverts to empty room in case of self chat", function() {
+                room.setSummary({
+                    "m.heroes": [],
+                    "m.invited_member_count": 1,
+                });
+                room.recalculate();
+                expect(room.name).toEqual("Empty room");
            });
        });

@@ -748,7 +791,7 @@ describe("Room", function() {
                addMember(userB);
                addMember(userC);
                addMember(userD);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                // we expect at least 1 member to be mentioned
                const others = [userB, userC, userD];
@@ -769,7 +812,7 @@ describe("Room", function() {
                addMember(userA);
                addMember(userB);
                addMember(userC);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
                expect(name.indexOf(userC)).toNotEqual(-1, name);
@@ -782,7 +825,7 @@ describe("Room", function() {
                addMember(userA);
                addMember(userB);
                addMember(userC);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
                expect(name.indexOf(userC)).toNotEqual(-1, name);
@@ -794,7 +837,7 @@ describe("Room", function() {
                setJoinRule("public");
                addMember(userA);
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -805,7 +848,7 @@ describe("Room", function() {
                setJoinRule("invite");
                addMember(userA);
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -815,7 +858,7 @@ describe("Room", function() {
                setJoinRule("invite");
                addMember(userA, "invite", {user: userB});
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -825,7 +868,7 @@ describe("Room", function() {
                const alias = "#room_alias:here";
                setJoinRule("invite");
                setAliases([alias, "#another:one"]);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
            });
@@ -835,7 +878,7 @@ describe("Room", function() {
                const alias = "#room_alias:here";
                setJoinRule("public");
                setAliases([alias, "#another:one"]);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
            });
@@ -845,7 +888,7 @@ describe("Room", function() {
                const roomName = "A mighty name indeed";
                setJoinRule("invite");
                setRoomName(roomName);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(roomName);
            });
@@ -855,25 +898,23 @@ describe("Room", function() {
                const roomName = "A mighty name indeed";
                setJoinRule("public");
                setRoomName(roomName);
-                room.recalculate(userA);
-                const name = room.name;
-                expect(name).toEqual(roomName);
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
            });

            it("should return 'Empty room' for private (invite join_rules) rooms if" +
            " a room name and alias don't exist and it is a self-chat.", function() {
                setJoinRule("invite");
                addMember(userA);
-                room.recalculate(userA);
-                const name = room.name;
-                expect(name).toEqual("Empty room");
+                room.recalculate();
+                expect(room.name).toEqual("Empty room");
            });

            it("should return 'Empty room' for public (public join_rules) rooms if a" +
            " room name and alias don't exist and it is a self-chat.", function() {
                setJoinRule("public");
                addMember(userA);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual("Empty room");
            });
@@ -881,30 +922,30 @@ describe("Room", function() {
            it("should return 'Empty room' if there is no name, " +
               "alias or members in the room.",
            function() {
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual("Empty room");
            });

-            it("should return 'Invite from [inviter display name] if state event " +
+            it("should return '[inviter display name] if state event " +
               "available",
            function() {
                setJoinRule("invite");
-                addMember(userA, 'join', {name: "Alice"});
-                addMember(userB, "invite", {user: userA});
-                room.recalculate(userB);
+                addMember(userB, 'join', {name: "Alice"});
+                addMember(userA, "invite", {user: userA});
+                room.recalculate();
                const name = room.name;
-                expect(name).toEqual("Invite from Alice");
+                expect(name).toEqual("Alice");
            });

            it("should return inviter mxid if display name not available",
            function() {
                setJoinRule("invite");
-                addMember(userA);
-                addMember(userB, "invite", {user: userA});
-                room.recalculate(userB);
+                addMember(userB);
+                addMember(userA, "invite", {user: userA});
+                room.recalculate();
                const name = room.name;
-                expect(name).toEqual("Invite from " + userA);
+                expect(name).toEqual(userB);
            });
        });
    });
@@ -1151,7 +1192,7 @@ describe("Room", function() {
    describe("addPendingEvent", function() {
        it("should add pending events to the pendingEventList if " +
                      "pendingEventOrdering == 'detached'", function() {
-            const room = new Room(roomId, {
+            const room = new Room(roomId, null, userA, {
                pendingEventOrdering: "detached",
            });
            const eventA = utils.mkMessage({
@@ -1177,7 +1218,7 @@ describe("Room", function() {

        it("should add pending events to the timeline if " +
                      "pendingEventOrdering == 'chronological'", function() {
-            room = new Room(roomId, {
+            room = new Room(roomId, null, userA, {
                pendingEventOrdering: "chronological",
            });
            const eventA = utils.mkMessage({
@@ -1201,7 +1242,7 @@ describe("Room", function() {

    describe("updatePendingEvent", function() {
        it("should remove cancelled events from the pending list", function() {
-            const room = new Room(roomId, {
+            const room = new Room(roomId, null, userA, {
                pendingEventOrdering: "detached",
            });
            const eventA = utils.mkMessage({
@@ -1237,7 +1278,7 @@ describe("Room", function() {


        it("should remove cancelled events from the timeline", function() {
-            const room = new Room(roomId);
+            const room = new Room(roomId, null, userA);
            const eventA = utils.mkMessage({
                room: roomId, user: userA, event: true,
            });
@@ -1269,4 +1310,153 @@ describe("Room", function() {
            expect(callCount).toEqual(1);
        });
    });
+
+    describe("loadMembersIfNeeded", function() {
+        function createClientMock(serverResponse, storageResponse = null) {
+            return {
+                getEventMapper: function() {
+                    // events should already be MatrixEvents
+                    return function(event) {return event;};
+                },
+                isRoomEncrypted: function() {
+                    return false;
+                },
+                _http: {
+                    serverResponse,
+                    authedRequest: function() {
+                        if (this.serverResponse instanceof Error) {
+                            return Promise.reject(this.serverResponse);
+                        } else {
+                            return Promise.resolve({chunk: this.serverResponse});
+                        }
+                    },
+                },
+                store: {
+                    storageResponse,
+                    storedMembers: null,
+                    getOutOfBandMembers: function() {
+                        if (this.storageResponse instanceof Error) {
+                            return Promise.reject(this.storageResponse);
+                        } else {
+                            return Promise.resolve(this.storageResponse);
+                        }
+                    },
+                    setOutOfBandMembers: function(roomId, memberEvents) {
+                        this.storedMembers = memberEvents;
+                        return Promise.resolve();
+                    },
+                    getSyncToken: () => "sync_token",
+                },
+            };
+        }
+
+        const memberEvent = utils.mkMembership({
+            user: "@user_a:bar", mship: "join",
+            room: roomId, event: true, name: "User A",
+        });
+
+        it("should load members from server on first call", async function() {
+            const client = createClientMock([memberEvent]);
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            await room.loadMembersIfNeeded();
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+            const storedMembers = client.store.storedMembers;
+            expect(storedMembers.length).toEqual(1);
+            expect(storedMembers[0].event_id).toEqual(memberEvent.getId());
+        });
+
+        it("should take members from storage if available", async function() {
+            const memberEvent2 = utils.mkMembership({
+                user: "@user_a:bar", mship: "join",
+                room: roomId, event: true, name: "Ms A",
+            });
+            const client = createClientMock([memberEvent2], [memberEvent]);
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+
+            await room.loadMembersIfNeeded();
+
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+        });
+
+        it("should allow retry on error", async function() {
+            const client = createClientMock(new Error("server says no"));
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            let hasThrown = false;
+            try {
+                await room.loadMembersIfNeeded();
+            } catch(err) {
+                hasThrown = true;
+            }
+            expect(hasThrown).toEqual(true);
+
+            client._http.serverResponse = [memberEvent];
+            await room.loadMembersIfNeeded();
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+        });
+    });
+
+    describe("getMyMembership", function() {
+        it("should return synced membership if membership isn't available yet",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.updateMyMembership("invite");
+            expect(room.getMyMembership()).toEqual("invite");
+        });
+        it("should emit a Room.myMembership event on a change",
+        function() {
+            const room = new Room(roomId, null, userA);
+            const events = [];
+            room.on("Room.myMembership", (_room, membership, oldMembership) => {
+                events.push({membership, oldMembership});
+            });
+            room.updateMyMembership("invite");
+            expect(room.getMyMembership()).toEqual("invite");
+            expect(events[0]).toEqual({membership: "invite", oldMembership: null});
+            events.splice(0);   //clear
+            room.updateMyMembership("invite");
+            expect(events.length).toEqual(0);
+            room.updateMyMembership("join");
+            expect(room.getMyMembership()).toEqual("join");
+            expect(events[0]).toEqual({membership: "join", oldMembership: "invite"});
+        });
+    });
+
+    describe("guessDMUserId", function() {
+        it("should return first hero id",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.setSummary({'m.heroes': [userB]});
+            expect(room.guessDMUserId()).toEqual(userB);
+        });
+        it("should return first member that isn't self",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.addLiveEvents([utils.mkMembership({
+                user: userB, mship: "join",
+                room: roomId, event: true,
+            })]);
+            expect(room.guessDMUserId()).toEqual(userB);
+        });
+        it("should return self if only member present",
+        function() {
+            const room = new Room(roomId, null, userA);
+            expect(room.guessDMUserId()).toEqual(userA);
+        });
+    });
+
+    describe("maySendMessage", function() {
+        it("should return false if synced membership not join",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.updateMyMembership("invite");
+            expect(room.maySendMessage()).toEqual(false);
+            room.updateMyMembership("leave");
+            expect(room.maySendMessage()).toEqual(false);
+            room.updateMyMembership("join");
+            expect(room.maySendMessage()).toEqual(true);
+        });
+    });
 });
@@ -2,7 +2,7 @@
 /* eslint new-cap: "off" */

 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const MatrixScheduler = sdk.MatrixScheduler;
 const MatrixError = sdk.MatrixError;
@@ -41,7 +41,7 @@ describe("MatrixScheduler", function() {
        });
        retryFn = null;
        queueFn = null;
-        defer = q.defer();
+        defer = Promise.defer();
    });

    afterEach(function() {
@@ -55,8 +55,8 @@ describe("MatrixScheduler", function() {
        queueFn = function() {
            return "one_big_queue";
        };
-        const deferA = q.defer();
-        const deferB = q.defer();
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
        let resolvedA = false;
        scheduler.setProcessFunction(function(event) {
            if (resolvedA) {
@@ -80,7 +80,7 @@ describe("MatrixScheduler", function() {
    it("should invoke the retryFn on failure and wait the amount of time specified",
    function(done) {
        const waitTimeMs = 1500;
-        const retryDefer = q.defer();
+        const retryDefer = Promise.defer();
        retryFn = function() {
            retryDefer.resolve();
            return waitTimeMs;
@@ -97,7 +97,7 @@ describe("MatrixScheduler", function() {
                return defer.promise;
            } else if (procCount === 2) {
                // don't care about this defer
-                return q.defer().promise;
+                return Promise.defer().promise;
            }
            expect(procCount).toBeLessThan(3);
        });
@@ -125,8 +125,8 @@ describe("MatrixScheduler", function() {
 return "yep";
 };

-        const deferA = q.defer();
-        const deferB = q.defer();
+        const deferA = Promise.defer();
+        const deferB = Promise.defer();
        let procCount = 0;
        scheduler.setProcessFunction(function(ev) {
            procCount += 1;
@@ -177,7 +177,7 @@ describe("MatrixScheduler", function() {
        const expectOrder = [
            eventA.getId(), eventB.getId(), eventD.getId(),
        ];
-        const deferA = q.defer();
+        const deferA = Promise.defer();
        scheduler.setProcessFunction(function(event) {
            const id = expectOrder.shift();
            expect(id).toEqual(event.getId());
@@ -52,6 +52,11 @@ describe("SyncAccumulator", function() {
                                member("bob", "join"),
                            ],
                        },
+                        summary: {
+                            "m.heroes": undefined,
+                            "m.joined_member_count": undefined,
+                            "m.invited_member_count": undefined,
+                        },
                        timeline: {
                            events: [msg("alice", "hi")],
                            prev_batch: "something",
@@ -318,6 +323,58 @@ describe("SyncAccumulator", function() {
            },
        });
    });
+
+    describe("summary field", function() {
+        function createSyncResponseWithSummary(summary) {
+            return {
+                next_batch: "abc",
+                rooms: {
+                    invite: {},
+                    leave: {},
+                    join: {
+                        "!foo:bar": {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            unread_notifications: {},
+                            state: {
+                                events: [],
+                            },
+                            summary: summary,
+                            timeline: {
+                                events: [],
+                                prev_batch: "something",
+                            },
+                        },
+                    },
+                },
+            };
+        }
+
+        it("should copy summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.heroes"]).toEqual(["@alice:bar"]);
+        });
+
+        it("should accumulate summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@bob:bar"],
+                "m.joined_member_count": 5,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.joined_member_count"]).toEqual(5);
+            expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
+        });
+    });
 });

 function syncSkeleton(joinObj) {
@@ -1,6 +1,6 @@
 "use strict";
 import 'source-map-support/register';
-const q = require("q");
+import Promise from 'bluebird';
 const sdk = require("../..");
 const EventTimeline = sdk.EventTimeline;
 const TimelineWindow = sdk.TimelineWindow;
@@ -157,7 +157,7 @@ describe("TimelineWindow", function() {
        client = {};
        client.getEventTimeline = function(timelineSet0, eventId0) {
            expect(timelineSet0).toBe(timelineSet);
-            return q(timeline);
+            return Promise.resolve(timeline);
        };

        return new TimelineWindow(client, timelineSet, opts);
@@ -179,7 +179,7 @@ describe("TimelineWindow", function() {
            timelineWindow.load(undefined, 2).then(function() {
                const expectedEvents = liveTimeline.getEvents().slice(1);
                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("should initialise from a specific event", function(done) {
@@ -191,14 +191,14 @@ describe("TimelineWindow", function() {
            client.getEventTimeline = function(timelineSet0, eventId0) {
                expect(timelineSet0).toBe(timelineSet);
                expect(eventId0).toEqual(eventId);
-                return q(timeline);
+                return Promise.resolve(timeline);
            };

            const timelineWindow = new TimelineWindow(client, timelineSet);
            timelineWindow.load(eventId, 3).then(function() {
                const expectedEvents = timeline.getEvents();
                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("canPaginate should return false until load has returned",
@@ -219,7 +219,7 @@ describe("TimelineWindow", function() {
                    .toBe(false);
                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                    .toBe(false);
-                return q(timeline);
+                return Promise.resolve(timeline);
            };

            timelineWindow.load(eventId, 3).then(function() {
@@ -229,7 +229,7 @@ describe("TimelineWindow", function() {
                    .toBe(true);
                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                    .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });
    });

@@ -277,7 +277,7 @@ describe("TimelineWindow", function() {
                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
            }).then(function(success) {
                expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("should advance into next timeline", function(done) {
@@ -322,7 +322,7 @@ describe("TimelineWindow", function() {
                return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
            }).then(function(success) {
                expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("should retreat into previous timeline", function(done) {
@@ -367,7 +367,7 @@ describe("TimelineWindow", function() {
                return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
            }).then(function(success) {
                expect(success).toBe(false);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("should make forward pagination requests", function(done) {
@@ -383,7 +383,7 @@ describe("TimelineWindow", function() {
                expect(opts.limit).toEqual(2);

                addEventsToTimeline(timeline, 3, false);
-                return q(true);
+                return Promise.resolve(true);
            };

            timelineWindow.load(eventId, 3).then(function() {
@@ -399,7 +399,7 @@ describe("TimelineWindow", function() {
                expect(success).toBe(true);
                const expectedEvents = timeline.getEvents().slice(0, 5);
                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });


@@ -416,7 +416,7 @@ describe("TimelineWindow", function() {
                expect(opts.limit).toEqual(2);

                addEventsToTimeline(timeline, 3, true);
-                return q(true);
+                return Promise.resolve(true);
            };

            timelineWindow.load(eventId, 3).then(function() {
@@ -432,7 +432,7 @@ describe("TimelineWindow", function() {
                expect(success).toBe(true);
                const expectedEvents = timeline.getEvents().slice(1, 6);
                expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });

        it("should limit the number of unsuccessful pagination requests",
@@ -449,7 +449,7 @@ describe("TimelineWindow", function() {
                expect(opts.backwards).toBe(false);
                expect(opts.limit).toEqual(2);
                paginateCount += 1;
-                return q(true);
+                return Promise.resolve(true);
            };

            timelineWindow.load(eventId, 3).then(function() {
@@ -471,7 +471,7 @@ describe("TimelineWindow", function() {
                    .toBe(false);
                expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                    .toBe(true);
-            }).catch(utils.failTest).done(done);
+            }).nodeify(done);
        });
    });
 });
@@ -0,0 +1,46 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module
+ */
+
+export default class Reemitter {
+    constructor(target) {
+        this.target = target;
+
+        // We keep one bound event handler for each event name so we know
+        // what event is arriving
+        this.boundHandlers = {};
+    }
+
+    _handleEvent(eventName, ...args) {
+        this.target.emit(eventName, ...args);
+    }
+
+    reEmit(source, eventNames) {
+        for (const eventName of eventNames) {
+            if (this.boundHandlers[eventName] === undefined) {
+                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
+            }
+            const boundHandler = this.boundHandlers[eventName];
+
+            source.on(eventName, boundHandler);
+        }
+    }
+}
@@ -0,0 +1,389 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/** @module auto-discovery */
+
+import Promise from 'bluebird';
+const logger = require("./logger");
+import { URL as NodeURL } from "url";
+
+// Dev note: Auto discovery is part of the spec.
+// See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
+
+/**
+ * Description for what an automatically discovered client configuration
+ * would look like. Although this is a class, it is recommended that it
+ * be treated as an interface definition rather than as a class.
+ *
+ * Additional properties than those defined here may be present, and
+ * should follow the Java package naming convention.
+ */
+class DiscoveredClientConfig { // eslint-disable-line no-unused-vars
+    // Dev note: this is basically a copy/paste of the .well-known response
+    // object as defined in the spec. It does have additional information,
+    // however. Overall, this exists to serve as a place for documentation
+    // and not functionality.
+    // See https://matrix.org/docs/spec/client_server/r0.4.0.html#get-well-known-matrix-client
+
+    constructor() {
+        /**
+         * The homeserver configuration the client should use. This will
+         * always be present on the object.
+         * @type {{state: string, base_url: string}} The configuration.
+         */
+        this["m.homeserver"] = {
+            /**
+             * The lookup result state. If this is anything other than
+             * AutoDiscovery.SUCCESS then base_url may be falsey. Additionally,
+             * if this is not AutoDiscovery.SUCCESS then the client should
+             * assume the other properties in the client config (such as
+             * the identity server configuration) are not valid.
+             */
+            state: AutoDiscovery.PROMPT,
+
+            /**
+             * If the state is AutoDiscovery.FAIL_ERROR or .FAIL_PROMPT
+             * then this will contain a human-readable (English) message
+             * for what went wrong. If the state is none of those previously
+             * mentioned, this will be falsey.
+             */
+            error: "Something went wrong",
+
+            /**
+             * The base URL clients should use to talk to the homeserver,
+             * particularly for the login process. May be falsey if the
+             * state is not AutoDiscovery.SUCCESS.
+             */
+            base_url: "https://matrix.org",
+        };
+
+        /**
+         * The identity server configuration the client should use. This
+         * will always be present on teh object.
+         * @type {{state: string, base_url: string}} The configuration.
+         */
+        this["m.identity_server"] = {
+            /**
+             * The lookup result state. If this is anything other than
+             * AutoDiscovery.SUCCESS then base_url may be falsey.
+             */
+            state: AutoDiscovery.PROMPT,
+
+            /**
+             * The base URL clients should use for interacting with the
+             * identity server. May be falsey if the state is not
+             * AutoDiscovery.SUCCESS.
+             */
+            base_url: "https://vector.im",
+        };
+    }
+}
+
+/**
+ * Utilities for automatically discovery resources, such as homeservers
+ * for users to log in to.
+ */
+export class AutoDiscovery {
+    // Dev note: the constants defined here are related to but not
+    // exactly the same as those in the spec. This is to hopefully
+    // translate the meaning of the states in the spec, but also
+    // support our own if needed.
+
+    /**
+     * The auto discovery failed. The client is expected to communicate
+     * the error to the user and refuse logging in.
+     * @return {string}
+     * @constructor
+     */
+    static get FAIL_ERROR() { return "FAIL_ERROR"; }
+
+    /**
+     * The auto discovery failed, however the client may still recover
+     * from the problem. The client is recommended to that the same
+     * action it would for PROMPT while also warning the user about
+     * what went wrong. The client may also treat this the same as
+     * a FAIL_ERROR state.
+     * @return {string}
+     * @constructor
+     */
+    static get FAIL_PROMPT() { return "FAIL_PROMPT"; }
+
+    /**
+     * The auto discovery didn't fail but did not find anything of
+     * interest. The client is expected to prompt the user for more
+     * information, or fail if it prefers.
+     * @return {string}
+     * @constructor
+     */
+    static get PROMPT() { return "PROMPT"; }
+
+    /**
+     * The auto discovery was successful.
+     * @return {string}
+     * @constructor
+     */
+    static get SUCCESS() { return "SUCCESS"; }
+
+    /**
+     * Attempts to automatically discover client configuration information
+     * prior to logging in. Such information includes the homeserver URL
+     * and identity server URL the client would want. Additional details
+     * may also be discovered, and will be transparently included in the
+     * response object unaltered.
+     * @param {string} domain The homeserver domain to perform discovery
+     * on. For example, "matrix.org".
+     * @return {Promise<DiscoveredClientConfig>} Resolves to the discovered
+     * configuration, which may include error states. Rejects on unexpected
+     * failure, not when discovery fails.
+     */
+    static async findClientConfig(domain) {
+        if (!domain || typeof(domain) !== "string" || domain.length === 0) {
+            throw new Error("'domain' must be a string of non-zero length");
+        }
+
+        // We use a .well-known lookup for all cases. According to the spec, we
+        // can do other discovery mechanisms if we want such as custom lookups
+        // however we won't bother with that here (mostly because the spec only
+        // supports .well-known right now).
+        //
+        // By using .well-known, we need to ensure we at least pull out a URL
+        // for the homeserver. We don't really need an identity server configuration
+        // but will return one anyways (with state PROMPT) to make development
+        // easier for clients. If we can't get a homeserver URL, all bets are
+        // off on the rest of the config and we'll assume it is invalid too.
+
+        // We default to an error state to make the first few checks easier to
+        // write. We'll update the properties of this object over the duration
+        // of this function.
+        const clientConfig = {
+            "m.homeserver": {
+                state: AutoDiscovery.FAIL_ERROR,
+                error: "Invalid homeserver discovery response",
+                base_url: null,
+            },
+            "m.identity_server": {
+                // Technically, we don't have a problem with the identity server
+                // config at this point.
+                state: AutoDiscovery.PROMPT,
+                error: null,
+                base_url: null,
+            },
+        };
+
+        // Step 1: Actually request the .well-known JSON file and make sure it
+        // at least has a homeserver definition.
+        const wellknown = await this._fetchWellKnownObject(
+            `https://${domain}/.well-known/matrix/client`,
+        );
+        if (!wellknown || wellknown.action !== "SUCCESS"
+            || !wellknown.raw["m.homeserver"]
+            || !wellknown.raw["m.homeserver"]["base_url"]) {
+            logger.error("No m.homeserver key in well-known response");
+            if (wellknown.reason) logger.error(wellknown.reason);
+            if (wellknown.action === "IGNORE") {
+                clientConfig["m.homeserver"] = {
+                    state: AutoDiscovery.PROMPT,
+                    error: null,
+                    base_url: null,
+                };
+            } else {
+                // this can only ever be FAIL_PROMPT at this point.
+                clientConfig["m.homeserver"].state = AutoDiscovery.FAIL_PROMPT;
+            }
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 2: Make sure the homeserver URL is valid *looking*. We'll make
+        // sure it points to a homeserver in Step 3.
+        const hsUrl = this._sanitizeWellKnownUrl(
+            wellknown.raw["m.homeserver"]["base_url"],
+        );
+        if (!hsUrl) {
+            logger.error("Invalid base_url for m.homeserver");
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 3: Make sure the homeserver URL points to a homeserver.
+        const hsVersions = await this._fetchWellKnownObject(
+            `${hsUrl}/_matrix/client/versions`,
+        );
+        if (!hsVersions || !hsVersions.raw["versions"]) {
+            logger.error("Invalid /versions response");
+            return Promise.resolve(clientConfig);
+        }
+
+        // Step 4: Now that the homeserver looks valid, update our client config.
+        clientConfig["m.homeserver"] = {
+            state: AutoDiscovery.SUCCESS,
+            error: null,
+            base_url: hsUrl,
+        };
+
+        // Step 5: Try to pull out the identity server configuration
+        let isUrl = "";
+        if (wellknown.raw["m.identity_server"]) {
+            // We prepare a failing identity server response to save lines later
+            // in this branch. Note that we also fail the homeserver check in the
+            // object because according to the spec we're supposed to FAIL_ERROR
+            // if *anything* goes wrong with the IS validation, including invalid
+            // format. This means we're supposed to stop discovery completely.
+            const failingClientConfig = {
+                "m.homeserver": {
+                    state: AutoDiscovery.FAIL_ERROR,
+                    error: "Invalid identity server discovery response",
+
+                    // We'll provide the base_url that was previously valid for
+                    // debugging purposes.
+                    base_url: clientConfig["m.homeserver"].base_url,
+                },
+                "m.identity_server": {
+                    state: AutoDiscovery.FAIL_ERROR,
+                    error: "Invalid identity server discovery response",
+                    base_url: null,
+                },
+            };
+
+            // Step 5a: Make sure the URL is valid *looking*. We'll make sure it
+            // points to an identity server in Step 5b.
+            isUrl = this._sanitizeWellKnownUrl(
+                wellknown.raw["m.identity_server"]["base_url"],
+            );
+            if (!isUrl) {
+                logger.error("Invalid base_url for m.identity_server");
+                return Promise.resolve(failingClientConfig);
+            }
+
+            // Step 5b: Verify there is an identity server listening on the provided
+            // URL.
+            const isResponse = await this._fetchWellKnownObject(
+                `${isUrl}/_matrix/identity/api/v1`,
+            );
+            if (!isResponse || !isResponse.raw || isResponse.action !== "SUCCESS") {
+                logger.error("Invalid /api/v1 response");
+                return Promise.resolve(failingClientConfig);
+            }
+        }
+
+        // Step 6: Now that the identity server is valid, or never existed,
+        // populate the IS section.
+        if (isUrl && isUrl.length > 0) {
+            clientConfig["m.identity_server"] = {
+                state: AutoDiscovery.SUCCESS,
+                error: null,
+                base_url: isUrl,
+            };
+        }
+
+        // Step 7: Copy any other keys directly into the clientConfig. This is for
+        // things like custom configuration of services.
+        Object.keys(wellknown.raw)
+            .filter((k) => k !== "m.homeserver" && k !== "m.identity_server")
+            .map((k) => clientConfig[k] = wellknown.raw[k]);
+
+        // Step 8: Give the config to the caller (finally)
+        return Promise.resolve(clientConfig);
+    }
+
+    /**
+     * Sanitizes a given URL to ensure it is either an HTTP or HTTP URL and
+     * is suitable for the requirements laid out by .well-known auto discovery.
+     * If valid, the URL will also be stripped of any trailing slashes.
+     * @param {string} url The potentially invalid URL to sanitize.
+     * @return {string|boolean} The sanitized URL or a falsey value if the URL is invalid.
+     * @private
+     */
+    static _sanitizeWellKnownUrl(url) {
+        if (!url) return false;
+
+        try {
+            // We have to try and parse the URL using the NodeJS URL
+            // library if we're on NodeJS and use the browser's URL
+            // library when we're in a browser. To accomplish this, we
+            // try the NodeJS version first and fall back to the browser.
+            let parsed = null;
+            try {
+                if (NodeURL) parsed = new NodeURL(url);
+                else parsed = new URL(url);
+            } catch (e) {
+                parsed = new URL(url);
+            }
+
+            if (!parsed || !parsed.hostname) return false;
+            if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return false;
+
+            const port = parsed.port ? `:${parsed.port}` : "";
+            const path = parsed.pathname ? parsed.pathname : "";
+            let saferUrl = `${parsed.protocol}//${parsed.hostname}${port}${path}`;
+            if (saferUrl.endsWith("/")) {
+                saferUrl = saferUrl.substring(0, saferUrl.length - 1);
+            }
+            return saferUrl;
+        } catch (e) {
+            logger.error(e);
+            return false;
+        }
+    }
+
+    /**
+     * Fetches a JSON object from a given URL, as expected by all .well-known
+     * related lookups. If the server gives a 404 then the `action` will be
+     * IGNORE. If the server returns something that isn't JSON, the `action`
+     * will be FAIL_PROMPT. For any other failure the `action` will be FAIL_PROMPT.
+     *
+     * The returned object will be a result of the call in object form with
+     * the following properties:
+     *   raw: The JSON object returned by the server.
+     *   action: One of SUCCESS, IGNORE, or FAIL_PROMPT.
+     *   reason: Relatively human readable description of what went wrong.
+     *   error: The actual Error, if one exists.
+     * @param {string} url The URL to fetch a JSON object from.
+     * @return {Promise<object>} Resolves to the returned state.
+     * @private
+     */
+    static async _fetchWellKnownObject(url) {
+        return new Promise(function(resolve, reject) {
+            const request = require("./matrix").getRequest();
+            if (!request) throw new Error("No request library available");
+            request(
+                { method: "GET", uri: url },
+                (err, response, body) => {
+                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                        let action = "FAIL_PROMPT";
+                        let reason = (err ? err.message : null) || "General failure";
+                        if (response.statusCode === 404) {
+                            action = "IGNORE";
+                            reason = "No .well-known JSON file found";
+                        }
+                        resolve({raw: {}, action: action, reason: reason, error: err});
+                        return;
+                    }
+
+                    try {
+                        resolve({raw: JSON.parse(body), action: "SUCCESS"});
+                    } catch (e) {
+                        let reason = "General failure";
+                        if (e.name === "SyntaxError") reason = "Invalid JSON";
+                        resolve({
+                            raw: {},
+                            action: "FAIL_PROMPT",
+                            reason: reason, error: e,
+                        });
+                    }
+                },
+            );
+        });
+    }
+}
@@ -54,6 +54,8 @@ const utils = require("./utils");
 * to all requests with this client. Useful for application services which require
 * <code>?user_id=</code>.
 *
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
 */
 function MatrixBaseApis(opts) {
    utils.checkObjectHasKeys(opts, ["baseUrl", "request"]);
@@ -70,6 +72,7 @@ function MatrixBaseApis(opts) {
        onlyData: true,
        extraParams: opts.queryParams,
        localTimeoutMs: opts.localTimeoutMs,
+        useAuthorizationHeader: opts.useAuthorizationHeader,
    };
    this._http = new httpApi.MatrixHttpApi(this, httpOpts);

@@ -86,9 +89,14 @@ MatrixBaseApis.prototype.getHomeserverUrl = function() {

 /**
 * Get the Identity Server URL of this client
+ * @param {boolean} stripProto whether or not to strip the protocol from the URL
 * @return {string} Identity Server URL of this client
 */
-MatrixBaseApis.prototype.getIdentityServerUrl = function() {
+MatrixBaseApis.prototype.getIdentityServerUrl = function(stripProto=false) {
+    if (stripProto && (this.idBaseUrl.startsWith("http://") ||
+            this.idBaseUrl.startsWith("https://"))) {
+        return this.idBaseUrl.split("://")[1];
+    }
    return this.idBaseUrl;
 };

@@ -254,7 +262,19 @@ MatrixBaseApis.prototype.login = function(loginType, data, callback) {
    utils.extend(login_data, data);

    return this._http.authedRequest(
-        callback, "POST", "/login", undefined, login_data,
+        (error, response) => {
+            if (loginType === "m.login.password" && response &&
+                response.access_token && response.user_id) {
+                this._http.opts.accessToken = response.access_token;
+                this.credentials = {
+                    userId: response.user_id,
+                };
+            }
+
+            if (callback) {
+                callback(error, response);
+            }
+        }, "POST", "/login", undefined, login_data,
    );
 };

@@ -290,9 +310,23 @@ MatrixBaseApis.prototype.loginWithSAML2 = function(relayState, callback) {
 * @return {string} The HS URL to hit to begin the CAS login process.
 */
 MatrixBaseApis.prototype.getCasLoginUrl = function(redirectUrl) {
-    return this._http.getUrl("/login/cas/redirect", {
+    return this.getSsoLoginUrl(redirectUrl, "cas");
+};
+
+/**
+ * @param {string} redirectUrl The URL to redirect to after the HS
+ *     authenticates with the SSO.
+ * @param {string} loginType The type of SSO login we are doing (sso or cas).
+ *     Defaults to 'sso'.
+ * @return {string} The HS URL to hit to begin the SSO login process.
+ */
+MatrixBaseApis.prototype.getSsoLoginUrl = function(redirectUrl, loginType) {
+    if (loginType === undefined) {
+        loginType = "sso";
+    }
+    return this._http.getUrl("/login/"+loginType+"/redirect", {
        "redirectUrl": redirectUrl,
-    }, httpApi.PREFIX_UNSTABLE);
+    }, httpApi.PREFIX_R0);
 };

 /**
@@ -330,18 +364,29 @@ MatrixBaseApis.prototype.logout = function(callback) {
 * it is up to the caller to either reset or destroy the MatrixClient after
 * this method succeeds.
 * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
- * @param {module:client.callback} callback Optional.
+ * @param {boolean} erase Optional. If set, send as `erase` attribute in the
+ * JSON request body, indicating whether the account should be erased. Defaults
+ * to false.
 * @return {module:client.Promise} Resolves: On success, the empty object
 */
-MatrixBaseApis.prototype.deactivateAccount = function(auth, callback) {
-    let body = {};
-    if (auth) {
-        body = {
-            auth: auth,
-        };
+MatrixBaseApis.prototype.deactivateAccount = function(auth, erase) {
+    if (typeof(erase) === 'function') {
+        throw new Error(
+            'deactivateAccount no longer accepts a callback parameter',
+        );
    }
+
+    const body = {};
+    if (auth) {
+        body.auth = auth;
+    }
+    if (erase !== undefined) {
+        body.erase = erase;
+    }
+
    return this._http.authedRequestWithPrefix(
-        callback, "POST", '/account/deactivate', undefined, body, httpApi.PREFIX_UNSTABLE,
+        undefined, "POST", '/account/deactivate', undefined, body,
+        httpApi.PREFIX_R0,
    );
 };

@@ -398,6 +443,402 @@ MatrixBaseApis.prototype.roomState = function(roomId, callback) {
    return this._http.authedRequest(callback, "GET", path);
 };

+/**
+ * Get an event in a room by its event id.
+ * @param {string} roomId
+ * @param {string} eventId
+ * @param {module:client.callback} callback Optional.
+ *
+ * @return {Promise} Resolves to an object containing the event.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.fetchRoomEvent = function(roomId, eventId, callback) {
+    const path = utils.encodeUri(
+        "/rooms/$roomId/event/$eventId", {
+            $roomId: roomId,
+            $eventId: eventId,
+        },
+    );
+    return this._http.authedRequest(callback, "GET", path);
+};
+
+/**
+ * @param {string} roomId
+ * @param {string} includeMembership the membership type to include in the response
+ * @param {string} excludeMembership the membership type to exclude from the response
+ * @param {string} atEventId the id of the event for which moment in the timeline the members should be returned for
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: dictionary of userid to profile information
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.members =
+function(roomId, includeMembership, excludeMembership, atEventId, callback) {
+    const queryParams = {};
+    if (includeMembership) {
+        queryParams.membership = includeMembership;
+    }
+    if (excludeMembership) {
+        queryParams.not_membership = excludeMembership;
+    }
+    if (atEventId) {
+        queryParams.at = atEventId;
+    }
+
+    const queryString = utils.encodeParams(queryParams);
+
+    const path = utils.encodeUri("/rooms/$roomId/members?" + queryString,
+        {$roomId: roomId});
+    return this._http.authedRequest(callback, "GET", path);
+};
+
+/**
+ * Upgrades a room to a new protocol version
+ * @param {string} roomId
+ * @param {string} newVersion The target version to upgrade to
+ * @return {module:client.Promise} Resolves: Object with key 'replacement_room'
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.upgradeRoom = function(roomId, newVersion) {
+    const path = utils.encodeUri("/rooms/$roomId/upgrade", {$roomId: roomId});
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, {new_version: newVersion},
+    );
+};
+
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group summary object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupSummary = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/summary", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group profile object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupProfile = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/profile", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @param {Object} profile The group profile object
+ * @param {string=} profile.name Name of the group
+ * @param {string=} profile.avatar_url MXC avatar URL
+ * @param {string=} profile.short_description A short description of the room
+ * @param {string=} profile.long_description A longer HTML description of the room
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupProfile = function(groupId, profile) {
+    const path = utils.encodeUri("/groups/$groupId/profile", {$groupId: groupId});
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, profile,
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {object} policy The join policy for the group. Must include at
+ *     least a 'type' field which is 'open' if anyone can join the group
+ *     the group without prior approval, or 'invite' if an invite is
+ *     required to join.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupJoinPolicy = function(groupId, policy) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/settings/m.join_policy",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(
+        undefined, "PUT", path, undefined, {
+            'm.join_policy': policy,
+        },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupInvitedUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/invited_users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group rooms list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupRooms = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/rooms", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.inviteUserToGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/invite/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroup = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/users/remove/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @param {string} roleId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addUserToGroupSummary = function(groupId, userId, roleId) {
+    const path = utils.encodeUri(
+        roleId ?
+            "/groups/$groupId/summary/$roleId/users/$userId" :
+            "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $roleId: roleId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroupSummary = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {string} categoryId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroupSummary = function(groupId, roomId, categoryId) {
+    const path = utils.encodeUri(
+        categoryId ?
+            "/groups/$groupId/summary/$categoryId/rooms/$roomId" :
+            "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $categoryId: categoryId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroupSummary = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroup = function(groupId, roomId, isPublic) {
+    if (isPublic === undefined) {
+        isPublic = true;
+    }
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { "m.visibility": { type: isPublic ? "public" : "private" } },
+    );
+};
+
+/**
+ * Configure the visibility of a room-group association.
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.updateGroupRoomVisibility = function(groupId, roomId, isPublic) {
+    // NB: The /config API is generic but there's not much point in exposing this yet as synapse
+    //     is the only server to implement this. In future we should consider an API that allows
+    //     arbitrary configuration, i.e. "config/$configKey".
+
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId/config/m.visibility",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { type: isPublic ? "public" : "private" },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroup = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {Object} opts Additional options to send alongside the acceptance.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.acceptGroupInvite = function(groupId, opts = null) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/accept_invite",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, opts || {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.joinGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/join",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.leaveGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/leave",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @return {module:client.Promise} Resolves: The groups to which the user is joined
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedGroups = function() {
+    const path = utils.encodeUri("/joined_groups");
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {Object} content Request content
+ * @param {string} content.localpart The local part of the desired group ID
+ * @param {Object} content.profile Group profile object
+ * @return {module:client.Promise} Resolves: Object with key group_id: id of the created group
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.createGroup = function(content) {
+    const path = utils.encodeUri("/create_group");
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, content,
+    );
+};
+
+/**
+ * @param {string[]} userIds List of user IDs
+ * @return {module:client.Promise} Resolves: Object as exmaple below
+ *
+ *     {
+ *         "users": {
+ *             "@bob:example.com": {
+ *                 "+example:example.com"
+ *             }
+ *         }
+ *     }
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getPublicisedGroups = function(userIds) {
+    const path = utils.encodeUri("/publicised_groups");
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, { user_ids: userIds },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {bool} isPublic Whether the user's membership of this group is made public
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupPublicity = function(groupId, isPublic) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/update_publicity",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {
+        publicise: isPublic,
+    });
+};
+
 /**
 * Retrieve a state event.
 * @param {string} roomId
@@ -512,6 +953,28 @@ MatrixBaseApis.prototype.setRoomReadMarkersHttpRequest =
    );
 };

+/**
+ * @return {module:client.Promise} Resolves: A list of the user's current rooms
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedRooms = function() {
+    const path = utils.encodeUri("/joined_rooms");
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * Retrieve membership info. for a room.
+ * @param {string} roomId ID of the room to get membership for
+ * @return {module:client.Promise} Resolves: A list of currently joined users
+ *                                 and their profile data.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedRoomMembers = function(roomId) {
+    const path = utils.encodeUri("/rooms/$roomId/joined_members", {
+        $roomId: roomId,
+    });
+    return this._http.authedRequest(undefined, "GET", path);
+};

 // Room Directory operations
 // =========================
@@ -719,6 +1182,10 @@ MatrixBaseApis.prototype.searchUserDirectory = function(opts) {
 * @param {string=} opts.name   Name to give the file on the server. Defaults
 *   to <tt>file.name</tt>.
 *
+ * @param {boolean=} opts.includeFilename if false will not send the filename,
+ *   e.g for encrypted file uploads where filename leaks are undesirable.
+ *   Defaults to true.
+ *
 * @param {string=} opts.type   Content-type for the upload. Defaults to
 *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
 *
@@ -735,6 +1202,10 @@ MatrixBaseApis.prototype.searchUserDirectory = function(opts) {
 *    invoke on success/failure. See the promise return values for more
 *    information.
 *
+ * @param {Function=} opts.progressHandler Optional. Called when a chunk of
+ *    data has been uploaded, with an object containing the fields `loaded`
+ *    (number of bytes transferred) and `total` (total size, if known).
+ *
 * @return {module:client.Promise} Resolves to response object, as
 *    determined by this.opts.onlyData, opts.rawResponse, and
 *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
@@ -924,6 +1395,27 @@ MatrixBaseApis.prototype.deleteDevice = function(device_id, auth) {
    );
 };

+/**
+ * Delete multiple device
+ *
+ * @param {string[]} devices IDs of the devices to delete
+ * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
+ * @return {module:client.Promise} Resolves: result object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.deleteMultipleDevices = function(devices, auth) {
+    const body = {devices};
+
+    if (auth) {
+        body.auth = auth;
+    }
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "POST", "/delete_devices", undefined, body,
+        httpApi.PREFIX_UNSTABLE,
+    );
+};
+

 // Push operations
 // ===============
@@ -1326,14 +1818,22 @@ MatrixBaseApis.prototype.getThirdpartyProtocols = function() {
    return this._http.authedRequestWithPrefix(
        undefined, "GET", "/thirdparty/protocols", undefined, undefined,
        httpApi.PREFIX_UNSTABLE,
-    );
+    ).then((response) => {
+        // sanity check
+        if (!response || typeof(response) !== 'object') {
+            throw new Error(
+                `/thirdparty/protocols did not return an object: ${response}`,
+            );
+        }
+        return response;
+    });
 };

 /**
 * Get information on how a specific place on a third party protocol
 * may be reached.
 * @param {string} protocol The protocol given in getThirdpartyProtocols()
- * @param {object} params Protocol-specific parameters, as given in th
+ * @param {object} params Protocol-specific parameters, as given in the
 *                        response to getThirdpartyProtocols()
 * @return {module:client.Promise} Resolves to the result object
 */
@@ -1348,6 +1848,25 @@ MatrixBaseApis.prototype.getThirdpartyLocation = function(protocol, params) {
    );
 };

+/**
+ * Get information on how a specific user on a third party protocol
+ * may be reached.
+ * @param {string} protocol The protocol given in getThirdpartyProtocols()
+ * @param {object} params Protocol-specific parameters, as given in the
+ *                        response to getThirdpartyProtocols()
+ * @return {module:client.Promise} Resolves to the result object
+ */
+MatrixBaseApis.prototype.getThirdpartyUser = function(protocol, params) {
+    const path = utils.encodeUri("/thirdparty/user/$protocol", {
+        $protocol: protocol,
+    });
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "GET", path, params, undefined,
+        httpApi.PREFIX_UNSTABLE,
+    );
+};
+
 /**
 * MatrixBaseApis object
 */
@@ -0,0 +1,100 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/** @module ContentHelpers */
+module.exports = {
+    /**
+     * Generates the content for a HTML Message event
+     * @param {string} body the plaintext body of the message
+     * @param {string} htmlBody the HTML representation of the message
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlMessage: function(body, htmlBody) {
+        return {
+            msgtype: "m.text",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Notice event
+     * @param {string} body the plaintext body of the notice
+     * @param {string} htmlBody the HTML representation of the notice
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlNotice: function(body, htmlBody) {
+        return {
+            msgtype: "m.notice",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Emote event
+     * @param {string} body the plaintext body of the emote
+     * @param {string} htmlBody the HTML representation of the emote
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlEmote: function(body, htmlBody) {
+        return {
+            msgtype: "m.emote",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Message event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeTextMessage: function(body) {
+        return {
+            msgtype: "m.text",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Notice event
+     * @param {string} body the plaintext body of the notice
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeNotice: function(body) {
+        return {
+            msgtype: "m.notice",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Emote event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeEmoteMessage: function(body) {
+        return {
+            msgtype: "m.emote",
+            body: body,
+        };
+    },
+};
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,13 +22,37 @@ limitations under the License.
 * Manages the list of other users' devices
 */

-import q from 'q';
+import Promise from 'bluebird';

+import logger from '../logger';
 import DeviceInfo from './deviceinfo';
 import olmlib from './olmlib';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+
+/* State transition diagram for DeviceList._deviceTrackingStatus
+ *
+ *                                |
+ *     stopTrackingDeviceList     V
+ *   +---------------------> NOT_TRACKED
+ *   |                            |
+ *   +<--------------------+      | startTrackingDeviceList
+ *   |                     |      V
+ *   |   +-------------> PENDING_DOWNLOAD <--------------------+-+
+ *   |   |                      ^ |                            | |
+ *   |   | restart     download | |  start download            | | invalidateUserDeviceList
+ *   |   | client        failed | |                            | |
+ *   |   |                      | V                            | |
+ *   |   +------------ DOWNLOAD_IN_PROGRESS -------------------+ |
+ *   |                    |       |                              |
+ *   +<-------------------+       |  download successful         |
+ *   ^                            V                              |
+ *   +----------------------- UP_TO_DATE ------------------------+
+ */
+

 // constants for DeviceList._deviceTrackingStatus
-// const TRACKING_STATUS_NOT_TRACKED = 0;
+const TRACKING_STATUS_NOT_TRACKED = 0;
 const TRACKING_STATUS_PENDING_DOWNLOAD = 1;
 const TRACKING_STATUS_DOWNLOAD_IN_PROGRESS = 2;
 const TRACKING_STATUS_UP_TO_DATE = 3;
@@ -36,31 +61,208 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
 * @alias module:crypto/DeviceList
 */
 export default class DeviceList {
-    constructor(baseApis, sessionStore, olmDevice) {
+    constructor(baseApis, cryptoStore, sessionStore, olmDevice) {
+        this._cryptoStore = cryptoStore;
        this._sessionStore = sessionStore;
-        this._serialiser = new DeviceListUpdateSerialiser(
-            baseApis, sessionStore, olmDevice,
-        );
+
+        // userId -> {
+        //     deviceId -> {
+        //         [device info]
+        //     }
+        // }
+        this._devices = {};
+
+        // map of identity keys to the user who owns it
+        this._userByIdentityKey = {};

        // which users we are tracking device status for.
        // userId -> TRACKING_STATUS_*
-        this._deviceTrackingStatus = sessionStore.getEndToEndDeviceTrackingStatus() || {};
+        this._deviceTrackingStatus = {}; // loaded from storage in load()
+
+        // The 'next_batch' sync token at the point the data was writen,
+        // ie. a token representing the point immediately after the
+        // moment represented by the snapshot in the db.
+        this._syncToken = null;
+
+        this._serialiser = new DeviceListUpdateSerialiser(
+            baseApis, olmDevice, this,
+        );
+
+        // userId -> promise
+        this._keyDownloadsInProgressByUser = {};
+
+        // Set whenever changes are made other than setting the sync token
+        this._dirty = false;
+
+        // Promise resolved when device data is saved
+        this._savePromise = null;
+        // Function that resolves the save promise
+        this._resolveSavePromise = null;
+        // The time the save is scheduled for
+        this._savePromiseTime = null;
+        // The timer used to delay the save
+        this._saveTimer = null;
+    }
+
+    /**
+     * Load the device tracking state from storage
+     */
+    async load() {
+        let shouldDeleteSessionStore = false;
+        await this._cryptoStore.doTxn(
+            // migrate from session store if there's data there and not here
+            'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    if (deviceData === null) {
+                        logger.log("Migrating e2e device data...");
+                        this._devices = this._sessionStore.getAllEndToEndDevices() || {};
+                        this._deviceTrackingStatus = (
+                            this._sessionStore.getEndToEndDeviceTrackingStatus() || {}
+                        );
+                        this._syncToken = this._sessionStore.getEndToEndDeviceSyncToken();
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                        shouldDeleteSessionStore = true;
+                    } else {
+                        this._devices = deviceData ? deviceData.devices : {},
+                        this._deviceTrackingStatus = deviceData ?
+                            deviceData.trackingStatus : {};
+                        this._syncToken = deviceData ? deviceData.syncToken : null;
+                    }
+                    this._userByIdentityKey = {};
+                    for (const user of Object.keys(this._devices)) {
+                        const userDevices = this._devices[user];
+                        for (const device of Object.keys(userDevices)) {
+                            const idKey = userDevices[device].keys['curve25519:'+device];
+                            if (idKey !== undefined) {
+                                this._userByIdentityKey[idKey] = user;
+                            }
+                        }
+                    }
+                });
+            },
+        );
+
+        if (shouldDeleteSessionStore) {
+            // migrated data is now safely persisted: remove from old store
+            this._sessionStore.removeEndToEndDeviceData();
+        }
+
        for (const u of Object.keys(this._deviceTrackingStatus)) {
            // if a download was in progress when we got shut down, it isn't any more.
            if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
                this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
            }
        }
+    }

-        // userId -> promise
-        this._keyDownloadsInProgressByUser = {};
-
-        this.lastKnownSyncToken = null;
+    stop() {
+        if (this._saveTimer !== null) {
+            clearTimeout(this._saveTimer);
+        }
    }

    /**
-     * Download the keys for a list of users and stores the keys in the session
-     * store.
+     * Save the device tracking state to storage, if any changes are
+     * pending other than updating the sync token
+     *
+     * The actual save will be delayed by a short amount of time to
+     * aggregate multiple writes to the database.
+     *
+     * @param {integer} delay Time in ms before which the save actually happens.
+     *     By default, the save is delayed for a short period in order to batch
+     *     multiple writes, but this behaviour can be disabled by passing 0.
+     *
+     * @return {Promise<bool>} true if the data was saved, false if
+     *     it was not (eg. because no changes were pending). The promise
+     *     will only resolve once the data is saved, so may take some time
+     *     to resolve.
+     */
+    async saveIfDirty(delay) {
+        if (!this._dirty) return Promise.resolve(false);
+        // Delay saves for a bit so we can aggregate multiple saves that happen
+        // in quick succession (eg. when a whole room's devices are marked as known)
+        if (delay === undefined) delay = 500;
+
+        const targetTime = Date.now + delay;
+        if (this._savePromiseTime && targetTime < this._savePromiseTime) {
+            // There's a save scheduled but for after we would like: cancel
+            // it & schedule one for the time we want
+            clearTimeout(this._saveTimer);
+            this._saveTimer = null;
+            this._savePromiseTime = null;
+            // (but keep the save promise since whatever called save before
+            // will still want to know when the save is done)
+        }
+
+        let savePromise = this._savePromise;
+        if (savePromise === null) {
+            savePromise = new Promise((resolve, reject) => {
+                this._resolveSavePromise = resolve;
+            });
+            this._savePromise = savePromise;
+        }
+
+        if (this._saveTimer === null) {
+            const resolveSavePromise = this._resolveSavePromise;
+            this._savePromiseTime = targetTime;
+            this._saveTimer = setTimeout(() => {
+                logger.log('Saving device tracking data at token ' + this._syncToken);
+                // null out savePromise now (after the delay but before the write),
+                // otherwise we could return the existing promise when the save has
+                // actually already happened. Likewise for the dirty flag.
+                this._savePromiseTime = null;
+                this._saveTimer = null;
+                this._savePromise = null;
+                this._resolveSavePromise = null;
+
+                this._dirty = false;
+                this._cryptoStore.doTxn(
+                    'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                    },
+                ).then(() => {
+                    resolveSavePromise();
+                });
+            }, delay);
+        }
+        return savePromise;
+    }
+
+    /**
+     * Gets the sync token last set with setSyncToken
+     *
+     * @return {string} The sync token
+     */
+    getSyncToken() {
+        return this._syncToken;
+    }
+
+    /**
+     * Sets the sync token that the app will pass as the 'since' to the /sync
+     * endpoint next time it syncs.
+     * The sync token must always be set after any changes made as a result of
+     * data in that sync since setting the sync token to a newer one will mean
+     * those changed will not be synced from the server if a new client starts
+     * up with that data.
+     *
+     * @param {string} st The sync token
+     */
+    setSyncToken(st) {
+        this._syncToken = st;
+    }
+
+    /**
+     * Ensures up to date keys for a list of users are stored in the session store,
+     * downloading and storing them if they're not (or if forceDownload is
+     * true).
     * @param {Array} userIds The users to fetch.
     * @param {bool} forceDownload Always download the keys even if cached.
     *
@@ -76,7 +278,7 @@ export default class DeviceList {
            if (this._keyDownloadsInProgressByUser[u]) {
                // already a key download in progress/queued for this user; its results
                // will be good enough for us.
-                console.log(
+                logger.log(
                    `downloadKeys: already have a download in progress for ` +
                    `${u}: awaiting its result`,
                );
@@ -87,16 +289,16 @@ export default class DeviceList {
        });

        if (usersToDownload.length != 0) {
-            console.log("downloadKeys: downloading for", usersToDownload);
+            logger.log("downloadKeys: downloading for", usersToDownload);
            const downloadPromise = this._doKeyDownload(usersToDownload);
            promises.push(downloadPromise);
        }

        if (promises.length === 0) {
-            console.log("downloadKeys: already have all necessary keys");
+            logger.log("downloadKeys: already have all necessary keys");
        }

-        return q.all(promises).then(() => {
+        return Promise.all(promises).then(() => {
            return this._getDevicesFromStore(userIds);
        });
    }
@@ -130,7 +332,7 @@ export default class DeviceList {
     * managed to get a list of devices for this user yet.
     */
    getStoredDevicesForUser(userId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
        if (!devs) {
            return null;
        }
@@ -143,6 +345,18 @@ export default class DeviceList {
        return res;
    }

+    /**
+     * Get the stored device data for a user, in raw object form
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @return {Object} deviceId->{object} devices, or undefined if
+     * there is no data for this user.
+     */
+    getRawStoredDevicesForUser(userId) {
+        return this._devices[userId];
+    }
+
    /**
     * Get the stored keys for a single device
     *
@@ -153,7 +367,7 @@ export default class DeviceList {
     * if we don't know about this device
     */
    getStoredDevice(userId, deviceId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
        if (!devs || !devs[deviceId]) {
            return undefined;
        }
@@ -163,13 +377,17 @@ export default class DeviceList {
    /**
     * Find a device by curve25519 identity key
     *
-     * @param {string} userId     owner of the device
     * @param {string} algorithm  encryption algorithm
     * @param {string} senderKey  curve25519 key to match
     *
     * @return {module:crypto/deviceinfo?}
     */
-    getDeviceByIdentityKey(userId, algorithm, senderKey) {
+    getDeviceByIdentityKey(algorithm, senderKey) {
+        const userId = this._userByIdentityKey[senderKey];
+        if (!userId) {
+            return null;
+        }
+
        if (
            algorithm !== olmlib.OLM_ALGORITHM &&
            algorithm !== olmlib.MEGOLM_ALGORITHM
@@ -178,7 +396,7 @@ export default class DeviceList {
            return null;
        }

-        const devices = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devices = this._devices[userId];
        if (!devices) {
            return null;
        }
@@ -207,6 +425,33 @@ export default class DeviceList {
        return null;
    }

+    /**
+     * Replaces the list of devices for a user with the given device list
+     *
+     * @param {string} u The user ID
+     * @param {Object} devs New device info for user
+     */
+    storeDevicesForUser(u, devs) {
+        // remove previous devices from _userByIdentityKey
+        if (this._devices[u] !== undefined) {
+            for (const [deviceId, dev] of Object.entries(this._devices[u])) {
+                const identityKey = dev.keys['curve25519:'+deviceId];
+
+                delete this._userByIdentityKey[identityKey];
+            }
+        }
+
+        this._devices[u] = devs;
+
+        // add new ones
+        for (const [deviceId, dev] of Object.entries(devs)) {
+            const identityKey = dev.keys['curve25519:'+deviceId];
+
+            this._userByIdentityKey[identityKey] = u;
+        }
+        this._dirty = true;
+    }
+
    /**
     * flag the given user for device-list tracking, if they are not already.
     *
@@ -228,12 +473,45 @@ export default class DeviceList {
            throw new Error('userId must be a string; was '+userId);
        }
        if (!this._deviceTrackingStatus[userId]) {
-            console.log('Now tracking device list for ' + userId);
+            logger.log('Now tracking device list for ' + userId);
            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
        }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
+    }
+
+    /**
+     * Mark the given user as no longer being tracked for device-list updates.
+     *
+     * This won't affect any in-progress downloads, which will still go on to
+     * complete; it will just mean that we don't think that we have an up-to-date
+     * list for future calls to downloadKeys.
+     *
+     * @param {String} userId
+     */
+    stopTrackingDeviceList(userId) {
+        if (this._deviceTrackingStatus[userId]) {
+            logger.log('No longer tracking device list for ' + userId);
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
+        }
+    }
+
+    /**
+     * Set all users we're currently tracking to untracked
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    stopTrackingAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        this._dirty = true;
    }

    /**
@@ -249,23 +527,12 @@ export default class DeviceList {
     */
    invalidateUserDeviceList(userId) {
        if (this._deviceTrackingStatus[userId]) {
-            console.log("Marking device list outdated for", userId);
+            logger.log("Marking device list outdated for", userId);
            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
-        }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
-    }

-    /**
-     * Mark all tracked device lists as outdated.
-     *
-     * This will flag each user whose devices we are tracking as in need of an
-     * update.
-     */
-    invalidateAllDeviceLists() {
-        for (const userId of Object.keys(this._deviceTrackingStatus)) {
-            this.invalidateUserDeviceList(userId);
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
        }
    }

@@ -276,6 +543,8 @@ export default class DeviceList {
     *    is no need to wait for this (it's mostly for the unit tests).
     */
    refreshOutdatedDeviceLists() {
+        this.saveIfDirty();
+
        const usersToDownload = [];
        for (const userId of Object.keys(this._deviceTrackingStatus)) {
            const stat = this._deviceTrackingStatus[userId];
@@ -283,17 +552,37 @@ export default class DeviceList {
                usersToDownload.push(userId);
            }
        }
-        if (usersToDownload.length == 0) {
-            return;
-        }
-
-        // we didn't persist the tracking status during
-        // invalidateUserDeviceList, so do it now.
-        this._persistDeviceTrackingStatus();

        return this._doKeyDownload(usersToDownload);
    }

+    /**
+     * Set the stored device data for a user, in raw object form
+     * Used only by internal class DeviceListUpdateSerialiser
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @param {Object} devices deviceId->{object} the new devices
+     */
+    _setRawStoredDevicesForUser(userId, devices) {
+        // remove old devices from _userByIdentityKey
+        if (this._devices[userId] !== undefined) {
+            for (const [deviceId, dev] of Object.entries(this._devices[userId])) {
+                const identityKey = dev.keys['curve25519:'+deviceId];
+
+                delete this._userByIdentityKey[identityKey];
+            }
+        }
+
+        this._devices[userId] = devices;
+
+        // add new devices into _userByIdentityKey
+        for (const [deviceId, dev] of Object.entries(devices)) {
+            const identityKey = dev.keys['curve25519:'+deviceId];
+
+            this._userByIdentityKey[identityKey] = userId;
+        }
+    }

    /**
     * Fire off download update requests for the given users, and update the
@@ -309,15 +598,15 @@ export default class DeviceList {
    _doKeyDownload(users) {
        if (users.length === 0) {
            // nothing to do
-            return q();
+            return Promise.resolve();
        }

        const prom = this._serialiser.updateDevicesForUsers(
-            users, this.lastKnownSyncToken,
+            users, this._syncToken,
        ).then(() => {
            finished(true);
        }, (e) => {
-            console.error(
+            logger.error(
                'Error downloading keys for ' + users + ":", e,
            );
            finished(false);
@@ -334,11 +623,13 @@ export default class DeviceList {

        const finished = (success) => {
            users.forEach((u) => {
+                this._dirty = true;
+
                // we may have queued up another download request for this user
                // since we started this request. If that happens, we should
                // ignore the completion of the first one.
                if (this._keyDownloadsInProgressByUser[u] !== prom) {
-                    console.log('Another update in the queue for', u,
+                    logger.log('Another update in the queue for', u,
                                '- not marking up-to-date');
                    return;
                }
@@ -349,21 +640,17 @@ export default class DeviceList {
                        // we didn't get any new invalidations since this download started:
                        // this user's device list is now up to date.
                        this._deviceTrackingStatus[u] = TRACKING_STATUS_UP_TO_DATE;
-                        console.log("Device list for", u, "now up to date");
+                        logger.log("Device list for", u, "now up to date");
                    } else {
                        this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
                    }
                }
            });
-            this._persistDeviceTrackingStatus();
+            this.saveIfDirty();
        };

        return prom;
    }
-
-    _persistDeviceTrackingStatus() {
-        this._sessionStore.storeEndToEndDeviceTrackingStatus(this._deviceTrackingStatus);
-    }
 }

 /**
@@ -376,10 +663,15 @@ export default class DeviceList {
 * time (and queuing other requests up).
 */
 class DeviceListUpdateSerialiser {
-    constructor(baseApis, sessionStore, olmDevice) {
+    /*
+     * @param {object} baseApis Base API object
+     * @param {object} olmDevice The Olm Device
+     * @param {object} deviceList The device list object
+     */
+    constructor(baseApis, olmDevice, deviceList) {
        this._baseApis = baseApis;
-        this._sessionStore = sessionStore;
        this._olmDevice = olmDevice;
+        this._deviceList = deviceList; // the device list to be updated

        this._downloadInProgress = false;

@@ -392,9 +684,7 @@ class DeviceListUpdateSerialiser {
        // non-null indicates that we have users queued for download.
        this._queuedQueryDeferred = null;

-        // sync token to be used for the next query: essentially the
-        // most recent one we know about
-        this._nextSyncToken = null;
+        this._syncToken = null; // The sync token we send with the requests
    }

    /**
@@ -413,15 +703,19 @@ class DeviceListUpdateSerialiser {
        users.forEach((u) => {
            this._keyDownloadsQueuedByUser[u] = true;
        });
-        this._nextSyncToken = syncToken;

        if (!this._queuedQueryDeferred) {
-            this._queuedQueryDeferred = q.defer();
+            this._queuedQueryDeferred = Promise.defer();
        }

+        // We always take the new sync token and just use the latest one we've
+        // been given, since it just needs to be at least as recent as the
+        // sync response the device invalidation message arrived in
+        this._syncToken = syncToken;
+
        if (this._downloadInProgress) {
            // just queue up these users
-            console.log('Queued key download for', users);
+            logger.log('Queued key download for', users);
            return this._queuedQueryDeferred.promise;
        }

@@ -441,12 +735,12 @@ class DeviceListUpdateSerialiser {
        const deferred = this._queuedQueryDeferred;
        this._queuedQueryDeferred = null;

-        console.log('Starting key download for', downloadUsers);
+        logger.log('Starting key download for', downloadUsers);
        this._downloadInProgress = true;

        const opts = {};
-        if (this._nextSyncToken) {
-            opts.token = this._nextSyncToken;
+        if (this._syncToken) {
+            opts.token = this._syncToken;
        }

        this._baseApis.downloadKeysForUsers(
@@ -459,16 +753,16 @@ class DeviceListUpdateSerialiser {
            //
            // of course we ought to do this in a web worker or similar, but
            // this serves as an easy solution for now.
-            let prom = q();
+            let prom = Promise.resolve();
            for (const userId of downloadUsers) {
                prom = prom.delay(5).then(() => {
-                    this._processQueryResponseForUser(userId, dk[userId]);
+                    return this._processQueryResponseForUser(userId, dk[userId]);
                });
            }

            return prom;
        }).done(() => {
-            console.log('Completed key download for ' + downloadUsers);
+            logger.log('Completed key download for ' + downloadUsers);

            this._downloadInProgress = false;
            deferred.resolve();
@@ -478,20 +772,20 @@ class DeviceListUpdateSerialiser {
                this._doQueuedQueries();
            }
        }, (e) => {
-            console.warn('Error downloading keys for ' + downloadUsers + ':', e);
-            this._downloadInProgressInProgress = false;
+            logger.warn('Error downloading keys for ' + downloadUsers + ':', e);
+            this._downloadInProgress = false;
            deferred.reject(e);
        });

        return deferred.promise;
    }

-    _processQueryResponseForUser(userId, response) {
-        console.log('got keys for ' + userId + ':', response);
+    async _processQueryResponseForUser(userId, response) {
+        logger.log('got keys for ' + userId + ':', response);

        // map from deviceid -> deviceinfo for this user
        const userStore = {};
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
        if (devs) {
            Object.keys(devs).forEach((deviceId) => {
                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
@@ -499,24 +793,22 @@ class DeviceListUpdateSerialiser {
            });
        }

-        _updateStoredDeviceKeysForUser(
+        await _updateStoredDeviceKeysForUser(
            this._olmDevice, userId, userStore, response || {},
        );

-        // update the session store
+        // put the updates into thr object that will be returned as our results
        const storage = {};
        Object.keys(userStore).forEach((deviceId) => {
            storage[deviceId] = userStore[deviceId].toStorage();
        });

-        this._sessionStore.storeEndToEndDevicesForUser(
-            userId, storage,
-        );
+        this._deviceList._setRawStoredDevicesForUser(userId, storage);
    }
 }


-function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
+async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        userResult) {
    let updated = false;

@@ -527,7 +819,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        }

        if (!(deviceId in userResult)) {
-            console.log("Device " + userId + ":" + deviceId +
+            logger.log("Device " + userId + ":" + deviceId +
                " has been removed");
            delete userStore[deviceId];
            updated = true;
@@ -544,17 +836,17 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        // check that the user_id and device_id in the response object are
        // correct
        if (deviceResult.user_id !== userId) {
-            console.warn("Mismatched user_id " + deviceResult.user_id +
+            logger.warn("Mismatched user_id " + deviceResult.user_id +
               " in keys from " + userId + ":" + deviceId);
            continue;
        }
        if (deviceResult.device_id !== deviceId) {
-            console.warn("Mismatched device_id " + deviceResult.device_id +
+            logger.warn("Mismatched device_id " + deviceResult.device_id +
               " in keys from " + userId + ":" + deviceId);
            continue;
        }

-        if (_storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
+        if (await _storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
            updated = true;
        }
    }
@@ -565,9 +857,9 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
 /*
 * Process a device in a /query response, and add it to the userStore
 *
- * returns true if a change was made, else false
+ * returns (a promise for) true if a change was made, else false
 */
- function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
+async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
    if (!deviceResult.keys) {
        // no keys?
        return false;
@@ -579,7 +871,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
    const signKeyId = "ed25519:" + deviceId;
    const signKey = deviceResult.keys[signKeyId];
    if (!signKey) {
-        console.warn("Device " + userId + ":" + deviceId +
+        logger.warn("Device " + userId + ":" + deviceId +
            " has no ed25519 key");
        return false;
    }
@@ -587,9 +879,9 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
    const unsigned = deviceResult.unsigned || {};

    try {
-        olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
+        await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
    } catch (e) {
-        console.warn("Unable to verify signature on device " +
+        logger.warn("Unable to verify signature on device " +
            userId + ":" + deviceId + ":" + e);
        return false;
    }
@@ -606,7 +898,7 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
            // best off sticking with the original keys.
            //
            // Should we warn the user about it somehow?
-            console.warn("Ed25519 key for device " + userId + ":" +
+            logger.warn("Ed25519 key for device " + userId + ":" +
               deviceId + " has changed");
            return false;
        }
@@ -0,0 +1,493 @@
+/*
+Copyright 2017 Vector Creations Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../logger';
+import utils from '../utils';
+
+/**
+ * Internal module. Management of outgoing room key requests.
+ *
+ * See https://docs.google.com/document/d/1m4gQkcnJkxNuBmb5NoFCIadIY-DyqqNAS3lloE73BlQ
+ * for draft documentation on what we're supposed to be implementing here.
+ *
+ * @module
+ */
+
+// delay between deciding we want some keys, and sending out the request, to
+// allow for (a) it turning up anyway, (b) grouping requests together
+const SEND_KEY_REQUESTS_DELAY_MS = 500;
+
+/** possible states for a room key request
+ *
+ * The state machine looks like:
+ *
+ *     |         (cancellation sent)
+ *     | .-------------------------------------------------.
+ *     | |                                                 |
+ *     V V       (cancellation requested)                  |
+ *   UNSENT  -----------------------------+                |
+ *     |                                  |                |
+ *     |                                  |                |
+ *     | (send successful)                |  CANCELLATION_PENDING_AND_WILL_RESEND
+ *     V                                  |                Λ
+ *    SENT                                |                |
+ *     |--------------------------------  |  --------------'
+ *     |                                  |  (cancellation requested with intent
+ *     |                                  |   to resend the original request)
+ *     |                                  |
+ *     | (cancellation requested)         |
+ *     V                                  |
+ * CANCELLATION_PENDING                   |
+ *     |                                  |
+ *     | (cancellation sent)              |
+ *     V                                  |
+ * (deleted)  <---------------------------+
+ *
+ * @enum {number}
+ */
+const ROOM_KEY_REQUEST_STATES = {
+    /** request not yet sent */
+    UNSENT: 0,
+
+    /** request sent, awaiting reply */
+    SENT: 1,
+
+    /** reply received, cancellation not yet sent */
+    CANCELLATION_PENDING: 2,
+
+    /**
+     * Cancellation not yet sent and will transition to UNSENT instead of
+     * being deleted once the cancellation has been sent.
+     */
+    CANCELLATION_PENDING_AND_WILL_RESEND: 3,
+};
+
+export default class OutgoingRoomKeyRequestManager {
+    constructor(baseApis, deviceId, cryptoStore) {
+        this._baseApis = baseApis;
+        this._deviceId = deviceId;
+        this._cryptoStore = cryptoStore;
+
+        // handle for the delayed call to _sendOutgoingRoomKeyRequests. Non-null
+        // if the callback has been set, or if it is still running.
+        this._sendOutgoingRoomKeyRequestsTimer = null;
+
+        // sanity check to ensure that we don't end up with two concurrent runs
+        // of _sendOutgoingRoomKeyRequests
+        this._sendOutgoingRoomKeyRequestsRunning = false;
+
+        this._clientRunning = false;
+    }
+
+    /**
+     * Called when the client is started. Sets background processes running.
+     */
+    start() {
+        this._clientRunning = true;
+
+        // set the timer going, to handle any requests which didn't get sent
+        // on the previous run of the client.
+        this._startTimer();
+    }
+
+    /**
+     * Called when the client is stopped. Stops any running background processes.
+     */
+    stop() {
+        logger.log('stopping OutgoingRoomKeyRequestManager');
+        // stop the timer on the next run
+        this._clientRunning = false;
+    }
+
+    /**
+     * Send off a room key request, if we haven't already done so.
+     *
+     * The `requestBody` is compared (with a deep-equality check) against
+     * previous queued or sent requests and if it matches, no change is made.
+     * Otherwise, a request is added to the pending list, and a job is started
+     * in the background to send it.
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {Array<{userId: string, deviceId: string}>} recipients
+     * @param {boolean} resend whether to resend the key request if there is
+     *    already one
+     *
+     * @returns {Promise} resolves when the request has been added to the
+     *    pending list (or we have established that a similar request already
+     *    exists)
+     */
+    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+        const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
+            requestBody,
+        );
+        if (!req) {
+            await this._cryptoStore.getOrAddOutgoingRoomKeyRequest({
+                requestBody: requestBody,
+                recipients: recipients,
+                requestId: this._baseApis.makeTxnId(),
+                state: ROOM_KEY_REQUEST_STATES.UNSENT,
+            });
+        } else {
+            switch (req.state) {
+            case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+            case ROOM_KEY_REQUEST_STATES.UNSENT:
+                // nothing to do here, since we're going to send a request anyways
+                return;
+
+            case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING: {
+                // existing request is about to be cancelled.  If we want to
+                // resend, then change the state so that it resends after
+                // cancelling.  Otherwise, just cancel the cancellation.
+                const state = resend ?
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
+                    ROOM_KEY_REQUEST_STATES.SENT;
+                await this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING, {
+                        state,
+                        cancellationTxnId: this._baseApis.makeTxnId(),
+                    },
+                );
+                break;
+            }
+            case ROOM_KEY_REQUEST_STATES.SENT: {
+                // a request has already been sent.  If we don't want to
+                // resend, then do nothing.  If we do want to, then cancel the
+                // existing request and send a new one.
+                if (resend) {
+                    const state =
+                          ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND;
+                    const updatedReq =
+                          await this._cryptoStore.updateOutgoingRoomKeyRequest(
+                              req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
+                                  state,
+                                  cancellationTxnId: this._baseApis.makeTxnId(),
+                                  // need to use a new transaction ID so that
+                                  // the request gets sent
+                                  requestTxnId: this._baseApis.makeTxnId(),
+                              },
+                          );
+                    if (!updatedReq) {
+                        // updateOutgoingRoomKeyRequest couldn't find the request
+                        // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
+                        // raced with another tab to mark the request cancelled.
+                        // Try again, to make sure the request is resent.
+                        return await this.sendRoomKeyRequest(
+                            requestBody, recipients, resend,
+                        );
+                    }
+
+                    // We don't want to wait for the timer, so we send it
+                    // immediately. (We might actually end up racing with the timer,
+                    // but that's ok: even if we make the request twice, we'll do it
+                    // with the same transaction_id, so only one message will get
+                    // sent).
+                    //
+                    // (We also don't want to wait for the response from the server
+                    // here, as it will slow down processing of received keys if we
+                    // do.)
+                    try {
+                        await this._sendOutgoingRoomKeyRequestCancellation(
+                            updatedReq,
+                            true,
+                        );
+                    } catch (e) {
+                        logger.error(
+                            "Error sending room key request cancellation;"
+                                + " will retry later.", e,
+                        );
+                    }
+                    // The request has transitioned from
+                    // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
+                    // still need to resend the request which is now UNSENT, so
+                    // start the timer if it isn't already started.
+                }
+                break;
+            }
+            default:
+                throw new Error('unhandled state: ' + req.state);
+            }
+        }
+        // some of the branches require the timer to be started.  Just start it
+        // all the time, because it doesn't hurt to start it.
+        this._startTimer();
+    }
+
+    /**
+     * Cancel room key requests, if any match the given requestBody
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *
+     * @returns {Promise} resolves when the request has been updated in our
+     *    pending list.
+     */
+    cancelRoomKeyRequest(requestBody) {
+        return this._cryptoStore.getOutgoingRoomKeyRequest(
+            requestBody,
+        ).then((req) => {
+            if (!req) {
+                // no request was made for this key
+                return;
+            }
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    // nothing to do here
+                    return;
+
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    // just delete it
+
+                    // FIXME: ghahah we may have attempted to send it, and
+                    // not yet got a successful response. So the server
+                    // may have seen it, so we still need to send a cancellation
+                    // in that case :/
+
+                    logger.log(
+                        'deleting unnecessary room key request for ' +
+                        stringifyRequestBody(requestBody),
+                    );
+                    return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                    );
+
+                case ROOM_KEY_REQUEST_STATES.SENT: {
+                    // send a cancellation.
+                    return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                        req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
+                            state: ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+                            cancellationTxnId: this._baseApis.makeTxnId(),
+                        },
+                    ).then((updatedReq) => {
+                        if (!updatedReq) {
+                            // updateOutgoingRoomKeyRequest couldn't find the
+                            // request in state ROOM_KEY_REQUEST_STATES.SENT,
+                            // so we must have raced with another tab to mark
+                            // the request cancelled. There is no point in
+                            // sending another cancellation since the other tab
+                            // will do it.
+                            logger.log(
+                                'Tried to cancel room key request for ' +
+                                stringifyRequestBody(requestBody) +
+                                ' but it was already cancelled in another tab',
+                            );
+                            return;
+                        }
+
+                        // We don't want to wait for the timer, so we send it
+                        // immediately. (We might actually end up racing with the timer,
+                        // but that's ok: even if we make the request twice, we'll do it
+                        // with the same transaction_id, so only one message will get
+                        // sent).
+                        //
+                        // (We also don't want to wait for the response from the server
+                        // here, as it will slow down processing of received keys if we
+                        // do.)
+                        this._sendOutgoingRoomKeyRequestCancellation(
+                            updatedReq,
+                        ).catch((e) => {
+                            logger.error(
+                                "Error sending room key request cancellation;"
+                                + " will retry later.", e,
+                            );
+                            this._startTimer();
+                        });
+                    });
+                }
+                default:
+                    throw new Error('unhandled state: ' + req.state);
+            }
+        });
+    }
+
+    /**
+     * Look for room key requests by target device and state
+     *
+     * @param {string} userId Target user ID
+     * @param {string} deviceId Target device ID
+     *
+     * @return {Promise} resolves to a list of all the
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     */
+    getOutgoingSentRoomKeyRequest(userId, deviceId) {
+        return this._cryptoStore.getOutgoingRoomKeyRequestsByTarget(
+            userId, deviceId, [ROOM_KEY_REQUEST_STATES.SENT],
+        );
+    }
+
+    // start the background timer to send queued requests, if the timer isn't
+    // already running
+    _startTimer() {
+        if (this._sendOutgoingRoomKeyRequestsTimer) {
+            return;
+        }
+
+        const startSendingOutgoingRoomKeyRequests = () => {
+            if (this._sendOutgoingRoomKeyRequestsRunning) {
+                throw new Error("RoomKeyRequestSend already in progress!");
+            }
+            this._sendOutgoingRoomKeyRequestsRunning = true;
+
+            this._sendOutgoingRoomKeyRequests().finally(() => {
+                this._sendOutgoingRoomKeyRequestsRunning = false;
+            }).catch((e) => {
+                // this should only happen if there is an indexeddb error,
+                // in which case we're a bit stuffed anyway.
+                logger.warn(
+                    `error in OutgoingRoomKeyRequestManager: ${e}`,
+                );
+            });
+        };
+
+        this._sendOutgoingRoomKeyRequestsTimer = global.setTimeout(
+            startSendingOutgoingRoomKeyRequests,
+            SEND_KEY_REQUESTS_DELAY_MS,
+        );
+    }
+
+    // look for and send any queued requests. Runs itself recursively until
+    // there are no more requests, or there is an error (in which case, the
+    // timer will be restarted before the promise resolves).
+    _sendOutgoingRoomKeyRequests() {
+        if (!this._clientRunning) {
+            this._sendOutgoingRoomKeyRequestsTimer = null;
+            return Promise.resolve();
+        }
+
+        logger.log("Looking for queued outgoing room key requests");
+
+        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+            ROOM_KEY_REQUEST_STATES.UNSENT,
+        ]).then((req) => {
+            if (!req) {
+                logger.log("No more outgoing room key requests");
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                return;
+            }
+
+            let prom;
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    prom = this._sendOutgoingRoomKeyRequest(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req, true);
+                    break;
+            }
+
+            return prom.then(() => {
+                // go around the loop again
+                return this._sendOutgoingRoomKeyRequests();
+            }).catch((e) => {
+                logger.error("Error sending room key request; will retry later.", e);
+                this._sendOutgoingRoomKeyRequestsTimer = null;
+                this._startTimer();
+            });
+        });
+    }
+
+    // given a RoomKeyRequest, send it and update the request record
+    _sendOutgoingRoomKeyRequest(req) {
+        logger.log(
+            `Requesting keys for ${stringifyRequestBody(req.requestBody)}` +
+            ` from ${stringifyRecipientList(req.recipients)}` +
+            `(id ${req.requestId})`,
+        );
+
+        const requestMessage = {
+            action: "request",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+            body: req.requestBody,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.requestTxnId || req.requestId,
+        ).then(() => {
+            return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
+                { state: ROOM_KEY_REQUEST_STATES.SENT },
+            );
+        });
+    }
+
+    // Given a RoomKeyRequest, cancel it and delete the request record unless
+    // andResend is set, in which case transition to UNSENT.
+    _sendOutgoingRoomKeyRequestCancellation(req, andResend) {
+        logger.log(
+            `Sending cancellation for key request for ` +
+            `${stringifyRequestBody(req.requestBody)} to ` +
+            `${stringifyRecipientList(req.recipients)} ` +
+            `(cancellation id ${req.cancellationTxnId})`,
+        );
+
+        const requestMessage = {
+            action: "request_cancellation",
+            requesting_device_id: this._deviceId,
+            request_id: req.requestId,
+        };
+
+        return this._sendMessageToDevices(
+            requestMessage, req.recipients, req.cancellationTxnId,
+        ).then(() => {
+            if (andResend) {
+                // We want to resend, so transition to UNSENT
+                return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId,
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+                    { state: ROOM_KEY_REQUEST_STATES.UNSENT },
+                );
+            }
+            return this._cryptoStore.deleteOutgoingRoomKeyRequest(
+                req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            );
+        });
+    }
+
+    // send a RoomKeyRequest to a list of recipients
+    _sendMessageToDevices(message, recipients, txnId) {
+        const contentMap = {};
+        for (const recip of recipients) {
+            if (!contentMap[recip.userId]) {
+                contentMap[recip.userId] = {};
+            }
+            contentMap[recip.userId][recip.deviceId] = message;
+        }
+
+        return this._baseApis.sendToDevice(
+            'm.room_key_request', contentMap, txnId,
+        );
+    }
+}
+
+function stringifyRequestBody(requestBody) {
+    // we assume that the request is for megolm keys, which are identified by
+    // room id and session id
+    return requestBody.room_id + " / " + requestBody.session_id;
+}
+
+function stringifyRecipientList(recipients) {
+    return '['
+        + utils.map(recipients, (r) => `${r.userId}:${r.deviceId}`).join(",")
+        + ']';
+}
+
@@ -0,0 +1,84 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module crypto/RoomList
+ *
+ * Manages the list of encrypted rooms
+ */
+
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+/**
+ * @alias module:crypto/RoomList
+ */
+export default class RoomList {
+    constructor(cryptoStore, sessionStore) {
+        this._cryptoStore = cryptoStore;
+        this._sessionStore = sessionStore;
+
+        // Object of roomId -> room e2e info object (body of the m.room.encryption event)
+        this._roomEncryption = {};
+    }
+
+    async init() {
+        let removeSessionStoreRooms = false;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.getEndToEndRooms(txn, (result) => {
+                    if (result === null || Object.keys(result).length === 0) {
+                        // migrate from session store, if there's data there
+                        const sessStoreRooms = this._sessionStore.getAllEndToEndRooms();
+                        if (sessStoreRooms !== null) {
+                            for (const roomId of Object.keys(sessStoreRooms)) {
+                                this._cryptoStore.storeEndToEndRoom(
+                                    roomId, sessStoreRooms[roomId], txn,
+                                );
+                            }
+                        }
+                        this._roomEncryption = sessStoreRooms;
+                        removeSessionStoreRooms = true;
+                    } else {
+                        this._roomEncryption = result;
+                    }
+                });
+            },
+        );
+        if (removeSessionStoreRooms) {
+            this._sessionStore.removeAllEndToEndRooms();
+        }
+    }
+
+    getRoomEncryption(roomId) {
+        return this._roomEncryption[roomId] || null;
+    }
+
+    isRoomEncrypted(roomId) {
+        return Boolean(this.getRoomEncryption(roomId));
+    }
+
+    async setRoomEncryption(roomId, roomInfo) {
+        // important that this happens before calling into the store
+        // as it prevents the Crypto::setRoomEncryption from calling
+        // this twice for consecutive m.room.encryption events
+        this._roomEncryption[roomId] = roomInfo;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+            },
+        );
+    }
+}
@@ -20,6 +20,8 @@ limitations under the License.
 * @module
 */

+import Promise from 'bluebird';
+
 /**
 * map of registered encryption algorithm classes. A map from string to {@link
 * module:crypto/algorithms/base.EncryptionAlgorithm|EncryptionAlgorithm} class
@@ -90,11 +92,11 @@ export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 * base type for decryption implementations
 *
 * @alias module:crypto/algorithms/base.DecryptionAlgorithm
- *
 * @param {object} params parameters
 * @param {string} params.userId  The UserID for the local user
 * @param {module:crypto} params.crypto crypto core
 * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
+ * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
 * @param {string=} params.roomId The ID of the room we will be receiving
 *     from. Null for to-device events.
 */
@@ -103,6 +105,7 @@ class DecryptionAlgorithm {
        this._userId = params.userId;
        this._crypto = params.crypto;
        this._olmDevice = params.olmDevice;
+        this._baseApis = params.baseApis;
        this._roomId = params.roomId;
    }

@@ -112,13 +115,11 @@ class DecryptionAlgorithm {
     * @method module:crypto/algorithms/base.DecryptionAlgorithm#decryptEvent
     * @abstract
     *
-     * @param {object} event raw event
+     * @param {MatrixEvent} event undecrypted event
     *
-     * @return {null} if the event referred to an unknown megolm session
-     * @return {module:crypto.DecryptionResult} decryption result
-     *
-     * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
-     *   problem decrypting the event
+     * @return {Promise<module:crypto~EventDecryptionResult>} promise which
+     * resolves once we have finished decrypting. Rejects with an
+     * `algorithms.DecryptionError` if there is a problem decrypting the event.
     */

    /**
@@ -140,6 +141,26 @@ class DecryptionAlgorithm {
    importRoomKey(session) {
        // ignore by default
    }
+
+    /**
+     * Determine if we have the keys necessary to respond to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     * @return {Promise<boolean>} true if we have the keys and could (theoretically) share
+     *  them; else false.
+     */
+    hasKeysForKeyRequest(keyRequest) {
+        return Promise.resolve(false);
+    }
+
+    /**
+     * Send the response to a room key request
+     *
+     * @param {module:crypto~IncomingRoomKeyRequest} keyRequest
+     */
+    shareKeysWithDevice(keyRequest) {
+        throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
+    }
 }
 export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272

@@ -155,34 +176,30 @@ export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 * @extends Error
 */
 class DecryptionError extends Error {
-    constructor(msg, details) {
+    constructor(code, msg, details) {
        super(msg);
+        this.code = code;
        this.name = 'DecryptionError';
-        this.details = details;
-    }
-
-    /**
-     * override the string used when logging
-     *
-     * @returns {String}
-     */
-    toString() {
-        let result = this.name + '[msg: ' + this.message;
-
-        if (this.details) {
-            result += ', ' +
-                Object.keys(this.details).map(
-                    (k) => k + ': ' + this.details[k],
-                ).join(', ');
-        }
-
-        result += ']';
-
-        return result;
+        this.detailedString = _detailedStringForDecryptionError(this, details);
    }
 }
 export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272

+function _detailedStringForDecryptionError(err, details) {
+    let result = err.name + '[msg: ' + err.message;
+
+    if (details) {
+        result += ', ' +
+            Object.keys(details).map(
+                (k) => k + ': ' + details[k],
+            ).join(', ');
+    }
+
+    result += ']';
+
+    return result;
+}
+
 /**
 * Exception thrown specifically when we want to warn the user to consider
 * the security of their conversation before continuing
@@ -20,8 +20,9 @@ limitations under the License.
 *
 * @module crypto/algorithms/olm
 */
-const q = require('q');
+import Promise from 'bluebird';

+const logger = require("../../logger");
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
 const DeviceInfo = require("../deviceinfo");
@@ -60,7 +61,7 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {

    if (this._sessionPrepared) {
        // prep already done
-        return q();
+        return Promise.resolve();
    }

    const self = this;
@@ -83,56 +84,62 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
 *
 * @return {module:client.Promise} Promise which resolves to the new event body
 */
-OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {
+OlmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
    // pick the list of recipients based on the membership list.
    //
    // TODO: there is a race condition here! What if a new user turns up
    // just as you are sending a secret message?

-    const users = utils.map(room.getJoinedMembers(), function(u) {
+    const members = await room.getEncryptionTargetMembers();
+
+    const users = utils.map(members, function(u) {
        return u.userId;
    });

    const self = this;
-    return this._ensureSession(users).then(function() {
-        const payloadFields = {
-            room_id: room.roomId,
-            type: eventType,
-            content: content,
-        };
+    await this._ensureSession(users);

-        const encryptedContent = {
-            algorithm: olmlib.OLM_ALGORITHM,
-            sender_key: self._olmDevice.deviceCurve25519Key,
-            ciphertext: {},
-        };
+    const payloadFields = {
+        room_id: room.roomId,
+        type: eventType,
+        content: content,
+    };

-        for (let i = 0; i < users.length; ++i) {
-            const userId = users[i];
-            const devices = self._crypto.getStoredDevicesForUser(userId);
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: self._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };

-            for (let j = 0; j < devices.length; ++j) {
-                const deviceInfo = devices[j];
-                const key = deviceInfo.getIdentityKey();
-                if (key == self._olmDevice.deviceCurve25519Key) {
-                    // don't bother sending to ourself
-                    continue;
-                }
-                if (deviceInfo.verified == DeviceVerification.BLOCKED) {
-                    // don't bother setting up sessions with blocked users
-                    continue;
-                }
+    const promises = [];

+    for (let i = 0; i < users.length; ++i) {
+        const userId = users[i];
+        const devices = self._crypto.getStoredDevicesForUser(userId);
+
+        for (let j = 0; j < devices.length; ++j) {
+            const deviceInfo = devices[j];
+            const key = deviceInfo.getIdentityKey();
+            if (key == self._olmDevice.deviceCurve25519Key) {
+                // don't bother sending to ourself
+                continue;
+            }
+            if (deviceInfo.verified == DeviceVerification.BLOCKED) {
+                // don't bother setting up sessions with blocked users
+                continue;
+            }
+
+            promises.push(
                olmlib.encryptMessageForDevice(
                    encryptedContent.ciphertext,
                    self._userId, self._deviceId, self._olmDevice,
                    userId, deviceInfo, payloadFields,
-                );
-            }
+                ),
+            );
        }
+    }

-        return encryptedContent;
-    });
+    return await Promise.all(promises).return(encryptedContent);
 };

 /**
@@ -153,28 +160,37 @@ utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
 *
 * @param {MatrixEvent} event
 *
- * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
- *   problem decrypting the event
+ * returns a promise which resolves to a
+ * {@link module:crypto~EventDecryptionResult} once we have finished
+ * decrypting. Rejects with an `algorithms.DecryptionError` if there is a
+ * problem decrypting the event.
 */
-OlmDecryption.prototype.decryptEvent = function(event) {
+OlmDecryption.prototype.decryptEvent = async function(event) {
    const content = event.getWireContent();
    const deviceKey = content.sender_key;
    const ciphertext = content.ciphertext;

    if (!ciphertext) {
-        throw new base.DecryptionError("Missing ciphertext");
+        throw new base.DecryptionError(
+            "OLM_MISSING_CIPHERTEXT",
+            "Missing ciphertext",
+        );
    }

    if (!(this._olmDevice.deviceCurve25519Key in ciphertext)) {
-        throw new base.DecryptionError("Not included in recipients");
+        throw new base.DecryptionError(
+            "OLM_NOT_INCLUDED_IN_RECIPIENTS",
+            "Not included in recipients",
+        );
    }
    const message = ciphertext[this._olmDevice.deviceCurve25519Key];
    let payloadString;

    try {
-        payloadString = this._decryptMessage(deviceKey, message);
+        payloadString = await this._decryptMessage(deviceKey, message);
    } catch (e) {
        throw new base.DecryptionError(
+            "OLM_BAD_ENCRYPTED_MESSAGE",
            "Bad Encrypted Message", {
                sender: deviceKey,
                err: e,
@@ -188,12 +204,14 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    // https://github.com/vector-im/vector-web/issues/2483
    if (payload.recipient != this._userId) {
        throw new base.DecryptionError(
+            "OLM_BAD_RECIPIENT",
            "Message was intented for " + payload.recipient,
        );
    }

    if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
        throw new base.DecryptionError(
+            "OLM_BAD_RECIPIENT_KEY",
            "Message not intended for this device", {
                intended: payload.recipient_keys.ed25519,
                our_key: this._olmDevice.deviceEd25519Key,
@@ -207,6 +225,7 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    // which is checked elsewhere).
    if (payload.sender != event.getSender()) {
        throw new base.DecryptionError(
+            "OLM_FORWARDED_MESSAGE",
            "Message forwarded from " + payload.sender, {
                reported_sender: event.getSender(),
            },
@@ -216,15 +235,21 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    // Olm events intended for a room have a room_id.
    if (payload.room_id !== event.getRoomId()) {
        throw new base.DecryptionError(
+            "OLM_BAD_ROOM",
            "Message intended for room " + payload.room_id, {
                reported_room: event.room_id,
            },
        );
    }

-    event.setClearData(payload, {curve25519: deviceKey}, payload.keys || {});
-};
+    const claimedKeys = payload.keys || {};

+    return {
+        clearEvent: payload,
+        senderCurve25519Key: deviceKey,
+        claimedEd25519Key: claimedKeys.ed25519 || null,
+    };
+};

 /**
 * Attempt to decrypt an Olm message
@@ -234,24 +259,28 @@ OlmDecryption.prototype.decryptEvent = function(event) {
 *
 * @return {string} payload, if decrypted successfully.
 */
-OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, message) {
-    const sessionIds = this._olmDevice.getSessionIdsForDevice(theirDeviceIdentityKey);
+OlmDecryption.prototype._decryptMessage = async function(
+    theirDeviceIdentityKey, message,
+) {
+    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
+        theirDeviceIdentityKey,
+    );

    // try each session in turn.
    const decryptionErrors = {};
    for (let i = 0; i < sessionIds.length; i++) {
        const sessionId = sessionIds[i];
        try {
-            const payload = this._olmDevice.decryptMessage(
+            const payload = await this._olmDevice.decryptMessage(
                theirDeviceIdentityKey, sessionId, message.type, message.body,
            );
-            console.log(
+            logger.log(
                "Decrypted Olm message from " + theirDeviceIdentityKey +
                    " with session " + sessionId,
            );
            return payload;
        } catch (e) {
-            const foundSession = this._olmDevice.matchesSession(
+            const foundSession = await this._olmDevice.matchesSession(
                theirDeviceIdentityKey, sessionId, message.type, message.body,
            );

@@ -289,7 +318,7 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa

    let res;
    try {
-        res = this._olmDevice.createInboundSession(
+        res = await this._olmDevice.createInboundSession(
            theirDeviceIdentityKey, message.type, message.body,
        );
    } catch (e) {
@@ -300,7 +329,7 @@ OlmDecryption.prototype._decryptMessage = function(theirDeviceIdentityKey, messa
        );
    }

-    console.log(
+    logger.log(
        "created new inbound Olm session ID " +
            res.session_id + " with " + theirDeviceIdentityKey,
    );
@@ -0,0 +1,81 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { randomString } from '../randomstring';
+
+const DEFAULT_ITERATIONS = 500000;
+
+export async function keyForExistingBackup(backupData, password) {
+    if (!global.Olm) {
+        throw new Error("Olm is not available");
+    }
+
+    const authData = backupData.auth_data;
+
+    if (!authData.private_key_salt || !authData.private_key_iterations) {
+        throw new Error(
+            "Salt and/or iterations not found: " +
+            "this backup cannot be restored with a passphrase",
+        );
+    }
+
+    return await deriveKey(
+        password, backupData.auth_data.private_key_salt,
+        backupData.auth_data.private_key_iterations,
+    );
+}
+
+export async function keyForNewBackup(password) {
+    if (!global.Olm) {
+        throw new Error("Olm is not available");
+    }
+
+    const salt = randomString(32);
+
+    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS);
+
+    return { key, salt, iterations: DEFAULT_ITERATIONS };
+}
+
+async function deriveKey(password, salt, iterations) {
+    const subtleCrypto = global.crypto.subtle;
+    const TextEncoder = global.TextEncoder;
+    if (!subtleCrypto || !TextEncoder) {
+        // TODO: Implement this for node
+        throw new Error("Password-based backup is not avaiable on this platform");
+    }
+
+    const key = await subtleCrypto.importKey(
+        'raw',
+        new TextEncoder().encode(password),
+        {name: 'PBKDF2'},
+        false,
+        ['deriveBits'],
+    );
+
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: 'PBKDF2',
+            salt: new TextEncoder().encode(salt),
+            iterations: iterations,
+            hash: 'SHA-512',
+        },
+        key,
+        global.Olm.PRIVATE_KEY_LENGTH * 8,
+    );
+
+    return new Uint8Array(keybits);
+}
@@ -20,9 +20,10 @@ limitations under the License.
 * Utilities common to olm encryption algorithms
 */

-const q = require('q');
+import Promise from 'bluebird';
 const anotherjson = require('another-json');

+const logger = require("../logger");
 const utils = require("../utils");

 /**
@@ -35,6 +36,11 @@ module.exports.OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2";
 */
 module.exports.MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";

+/**
+ * matrix algorithm tag for megolm backups
+ */
+module.exports.MEGOLM_BACKUP_ALGORITHM = "m.megolm_backup.v1.curve25519-aes-sha2";
+

 /**
 * Encrypt an event payload for an Olm device
@@ -48,21 +54,24 @@ module.exports.MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";
 * @param {string} recipientUserId
 * @param {module:crypto/deviceinfo} recipientDevice
 * @param {object} payloadFields fields to include in the encrypted payload
+ *
+ * Returns a promise which resolves (to undefined) when the payload
+ *    has been encrypted into `resultsObject`
 */
-module.exports.encryptMessageForDevice = function(
+module.exports.encryptMessageForDevice = async function(
    resultsObject,
    ourUserId, ourDeviceId, olmDevice, recipientUserId, recipientDevice,
    payloadFields,
 ) {
    const deviceKey = recipientDevice.getIdentityKey();
-    const sessionId = olmDevice.getSessionIdForDevice(deviceKey);
+    const sessionId = await olmDevice.getSessionIdForDevice(deviceKey);
    if (sessionId === null) {
        // If we don't have a session for a device then
        // we can't encrypt a message for it.
        return;
    }

-    console.log(
+    logger.log(
        "Using sessionid " + sessionId + " for device " +
            recipientUserId + ":" + recipientDevice.deviceId,
    );
@@ -99,7 +108,7 @@ module.exports.encryptMessageForDevice = function(

    utils.extend(payload, payloadFields);

-    resultsObject[deviceKey] = olmDevice.encryptMessage(
+    resultsObject[deviceKey] = await olmDevice.encryptMessage(
        deviceKey, sessionId, JSON.stringify(payload),
    );
 };
@@ -112,14 +121,17 @@ module.exports.encryptMessageForDevice = function(
 * @param {module:base-apis~MatrixBaseApis} baseApis
 *
 * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
- *    map from userid to list of devices
+ *    map from userid to list of devices to ensure sessions for
+ *
+ * @param {bolean} force If true, establish a new session even if one already exists.
+ *     Optional.
 *
 * @return {module:client.Promise} resolves once the sessions are complete, to
 *    an Object mapping from userId to deviceId to
 *    {@link module:crypto~OlmSessionResult}
 */
-module.exports.ensureOlmSessionsForDevices = function(
-    olmDevice, baseApis, devicesByUser,
+module.exports.ensureOlmSessionsForDevices = async function(
+    olmDevice, baseApis, devicesByUser, force,
 ) {
    const devicesWithoutSession = [
        // [userId, deviceId], ...
@@ -136,8 +148,8 @@ module.exports.ensureOlmSessionsForDevices = function(
            const deviceInfo = devices[j];
            const deviceId = deviceInfo.deviceId;
            const key = deviceInfo.getIdentityKey();
-            const sessionId = olmDevice.getSessionIdForDevice(key);
-            if (sessionId === null) {
+            const sessionId = await olmDevice.getSessionIdForDevice(key);
+            if (sessionId === null || force) {
                devicesWithoutSession.push([userId, deviceId]);
            }
            result[userId][deviceId] = {
@@ -148,7 +160,7 @@ module.exports.ensureOlmSessionsForDevices = function(
    }

    if (devicesWithoutSession.length === 0) {
-        return q(result);
+        return result;
    }

    // TODO: this has a race condition - if we try to send another message
@@ -158,60 +170,65 @@ module.exports.ensureOlmSessionsForDevices = function(
    // That should eventually resolve itself, but it's poor form.

    const oneTimeKeyAlgorithm = "signed_curve25519";
-    return baseApis.claimOneTimeKeys(
+    const res = await baseApis.claimOneTimeKeys(
        devicesWithoutSession, oneTimeKeyAlgorithm,
-    ).then(function(res) {
-        const otk_res = res.one_time_keys || {};
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
+    );
+
+    const otk_res = res.one_time_keys || {};
+    const promises = [];
+    for (const userId in devicesByUser) {
+        if (!devicesByUser.hasOwnProperty(userId)) {
+            continue;
+        }
+        const userRes = otk_res[userId] || {};
+        const devices = devicesByUser[userId];
+        for (let j = 0; j < devices.length; j++) {
+            const deviceInfo = devices[j];
+            const deviceId = deviceInfo.deviceId;
+            if (result[userId][deviceId].sessionId && !force) {
+                // we already have a result for this device
                continue;
            }
-            const userRes = otk_res[userId] || {};
-            const devices = devicesByUser[userId];
-            for (let j = 0; j < devices.length; j++) {
-                const deviceInfo = devices[j];
-                const deviceId = deviceInfo.deviceId;
-                if (result[userId][deviceId].sessionId) {
-                    // we already have a result for this device
-                    continue;
-                }

-                const deviceRes = userRes[deviceId] || {};
-                let oneTimeKey = null;
-                for (const keyId in deviceRes) {
-                    if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
-                        oneTimeKey = deviceRes[keyId];
-                    }
+            const deviceRes = userRes[deviceId] || {};
+            let oneTimeKey = null;
+            for (const keyId in deviceRes) {
+                if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
+                    oneTimeKey = deviceRes[keyId];
                }
-
-                if (!oneTimeKey) {
-                    console.warn(
-                        "No one-time keys (alg=" + oneTimeKeyAlgorithm +
-                            ") for device " + userId + ":" + deviceId,
-                    );
-                    continue;
-                }
-
-                const sid = _verifyKeyAndStartSession(
-                    olmDevice, oneTimeKey, userId, deviceInfo,
-                );
-                result[userId][deviceId].sessionId = sid;
            }
+
+            if (!oneTimeKey) {
+                logger.warn(
+                    "No one-time keys (alg=" + oneTimeKeyAlgorithm +
+                        ") for device " + userId + ":" + deviceId,
+                );
+                continue;
+            }
+
+            promises.push(
+                _verifyKeyAndStartSession(
+                    olmDevice, oneTimeKey, userId, deviceInfo,
+                ).then((sid) => {
+                    result[userId][deviceId].sessionId = sid;
+                }),
+            );
        }
-        return result;
-    });
+    }
+
+    await Promise.all(promises);
+    return result;
 };

-
-function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
+async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
    const deviceId = deviceInfo.deviceId;
    try {
-        _verifySignature(
+        await _verifySignature(
            olmDevice, oneTimeKey, userId, deviceId,
            deviceInfo.getFingerprint(),
        );
    } catch (e) {
-        console.error(
+        logger.error(
            "Unable to verify signature on one-time key for device " +
                userId + ":" + deviceId + ":", e,
        );
@@ -220,17 +237,17 @@ function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {

    let sid;
    try {
-        sid = olmDevice.createOutboundSession(
+        sid = await olmDevice.createOutboundSession(
            deviceInfo.getIdentityKey(), oneTimeKey.key,
        );
    } catch (e) {
        // possibly a bad key
-        console.error("Error starting session with device " +
+        logger.error("Error starting session with device " +
                      userId + ":" + deviceId + ": " + e);
        return null;
    }

-    console.log("Started new sessionid " + sid +
+    logger.log("Started new sessionid " + sid +
                " for device " + userId + ":" + deviceId);
    return sid;
 }
@@ -249,8 +266,11 @@ function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
 * @param {string} signingDeviceId  ID of the device whose signature should be checked
 *
 * @param {string} signingKey   base64-ed ed25519 public key
+ *
+ * Returns a promise which resolves (to undefined) if the the signature is good,
+ * or rejects with an Error if it is bad.
 */
-const _verifySignature = module.exports.verifySignature = function(
+const _verifySignature = module.exports.verifySignature = async function(
    olmDevice, obj, signingUserId, signingDeviceId, signingKey,
 ) {
    const signKeyId = "ed25519:" + signingDeviceId;
@@ -0,0 +1,66 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import bs58 from 'bs58';
+
+// picked arbitrarily but to try & avoid clashing with any bitcoin ones
+// (which are also base58 encoded, but bitcoin's involve a lot more hashing)
+const OLM_RECOVERY_KEY_PREFIX = [0x8B, 0x01];
+
+export function encodeRecoveryKey(key) {
+    const buf = new Uint8Array(OLM_RECOVERY_KEY_PREFIX.length + key.length + 1);
+    buf.set(OLM_RECOVERY_KEY_PREFIX, 0);
+    buf.set(key, OLM_RECOVERY_KEY_PREFIX.length);
+
+    let parity = 0;
+    for (let i = 0; i < buf.length - 1; ++i) {
+        parity ^= buf[i];
+    }
+    buf[buf.length - 1] = parity;
+    const base58key = bs58.encode(buf);
+
+    return base58key.match(/.{1,4}/g).join(" ");
+}
+
+export function decodeRecoveryKey(recoverykey) {
+    const result = bs58.decode(recoverykey.replace(/ /g, ''));
+
+    let parity = 0;
+    for (const b of result) {
+        parity ^= b;
+    }
+    if (parity !== 0) {
+        throw new Error("Incorrect parity");
+    }
+
+    for (let i = 0; i < OLM_RECOVERY_KEY_PREFIX.length; ++i) {
+        if (result[i] !== OLM_RECOVERY_KEY_PREFIX[i]) {
+            throw new Error("Incorrect prefix");
+        }
+    }
+
+    if (
+        result.length !==
+        OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH + 1
+    ) {
+        throw new Error("Incorrect length");
+    }
+
+    return result.slice(
+        OLM_RECOVERY_KEY_PREFIX.length,
+        OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH,
+    );
+}
@@ -9,3 +9,26 @@
 *
 * @interface CryptoStore
 */
+
+/**
+ * Represents an outgoing room key request
+ *
+ * @typedef {Object} OutgoingRoomKeyRequest
+ *
+ * @property {string} requestId    unique id for this request. Used for both
+ *    an id within the request for later pairing with a cancellation, and for
+ *    the transaction id when sending the to_device messages to our local
+ *    server.
+ *
+ * @property {string?} cancellationTxnId
+ *    transaction id for the cancellation, if any
+ *
+ * @property {Array<{userId: string, deviceId: string}>} recipients
+ *    list of recipients for the request
+ *
+ * @property {module:crypto~RoomKeyRequestBody} requestBody
+ *    parameters for the request.
+ *
+ * @property {Number} state   current state of this request (states are defined
+ *    in {@link module:crypto/OutgoingRoomKeyRequestManager~ROOM_KEY_REQUEST_STATES})
+ */
@@ -0,0 +1,702 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import utils from '../../utils';
+
+export const VERSION = 7;
+
+/**
+ * Implementation of a CryptoStore which is backed by an existing
+ * IndexedDB connection. Generally you want IndexedDBCryptoStore
+ * which connects to the database and defers to one of these.
+ *
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export class Backend {
+    /**
+     * @param {IDBDatabase} db
+     */
+    constructor(db) {
+        this._db = db;
+
+        // make sure we close the db on `onversionchange` - otherwise
+        // attempts to delete the database will block (and subsequent
+        // attempts to re-create it will also block).
+        db.onversionchange = (ev) => {
+            logger.log(`versionchange for indexeddb ${this._dbName}: closing`);
+            db.close();
+        };
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        const deferred = Promise.defer();
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        txn.onerror = deferred.reject;
+
+        // first see if we already have an entry for this request.
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            if (existing) {
+                // this entry matches the request - return it.
+                logger.log(
+                    `already have key request outstanding for ` +
+                        `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                        `not sending another`,
+                );
+                deferred.resolve(existing);
+                return;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            logger.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                    requestBody.session_id,
+            );
+            txn.oncomplete = () => { deferred.resolve(request); };
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            store.add(request);
+        });
+
+        return deferred.promise;
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        const deferred = Promise.defer();
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        txn.onerror = deferred.reject;
+
+        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
+            deferred.resolve(existing);
+        });
+        return deferred.promise;
+    }
+
+    /**
+     * look for an existing room key request in the db
+     *
+     * @private
+     * @param {IDBTransaction} txn  database transaction
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     * @param {Function} callback  function to call with the results of the
+     *    search. Either passed a matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found.
+     */
+    _getOutgoingRoomKeyRequest(txn, requestBody, callback) {
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const idx = store.index("session");
+        const cursorReq = idx.openCursor([
+            requestBody.room_id,
+            requestBody.session_id,
+        ]);
+
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if(!cursor) {
+                // no match found
+                callback(null);
+                return;
+            }
+
+            const existing = cursor.value;
+
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                // got a match
+                callback(existing);
+                return;
+            }
+
+            // look at the next entry in the index
+            cursor.continue();
+        };
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        if (wantedStates.length === 0) {
+            return Promise.resolve(null);
+        }
+
+        // this is a bit tortuous because we need to make sure we do the lookup
+        // in a single transaction, to avoid having a race with the insertion
+        // code.
+
+        // index into the wantedStates array
+        let stateIndex = 0;
+        let result;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (cursor) {
+                // got a match
+                result = cursor.value;
+                return;
+            }
+
+            // try the next state in the list
+            stateIndex++;
+            if (stateIndex >= wantedStates.length) {
+                // no matches
+                return;
+            }
+
+            const wantedState = wantedStates[stateIndex];
+            const cursorReq = ev.target.source.openCursor(wantedState);
+            cursorReq.onsuccess = onsuccess;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const wantedState = wantedStates[stateIndex];
+        const cursorReq = store.index("state").openCursor(wantedState);
+        cursorReq.onsuccess = onsuccess;
+
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
+        let stateIndex = 0;
+        const results = [];
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (cursor) {
+                const keyReq = cursor.value;
+                if (keyReq.recipients.includes({userId, deviceId})) {
+                    results.push(keyReq);
+                }
+                cursor.continue();
+            } else {
+                // try the next state in the list
+                stateIndex++;
+                if (stateIndex >= wantedStates.length) {
+                    // no matches
+                    return;
+                }
+
+                const wantedState = wantedStates[stateIndex];
+                const cursorReq = ev.target.source.openCursor(wantedState);
+                cursorReq.onsuccess = onsuccess;
+            }
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+        const store = txn.objectStore("outgoingRoomKeyRequests");
+
+        const wantedState = wantedStates[stateIndex];
+        const cursorReq = store.index("state").openCursor(wantedState);
+        cursorReq.onsuccess = onsuccess;
+
+        return promiseifyTxn(txn).then(() => results);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        let result = null;
+
+        function onsuccess(ev) {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                logger.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${data.state}`,
+                );
+                return;
+            }
+            Object.assign(data, updates);
+            cursor.update(data);
+            result = data;
+        }
+
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = onsuccess;
+        return promiseifyTxn(txn).then(() => result);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        const txn = this._db.transaction("outgoingRoomKeyRequests", "readwrite");
+        const cursorReq = txn.objectStore("outgoingRoomKeyRequests")
+                  .openCursor(requestId);
+        cursorReq.onsuccess = (ev) => {
+            const cursor = ev.target.result;
+            if (!cursor) {
+                return;
+            }
+            const data = cursor.value;
+            if (data.state != expectedState) {
+                logger.warn(
+                    `Cannot delete room key request in state ${data.state} `
+                        + `(expected ${expectedState})`,
+                );
+                return;
+            }
+            cursor.delete();
+        };
+        return promiseifyTxn(txn);
+    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeAccount(txn, newData) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(newData, "-");
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const countReq = objectStore.count();
+        countReq.onsuccess = function() {
+            func(countReq.result);
+        };
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const idx = objectStore.index("deviceKey");
+        const getReq = idx.openCursor(deviceKey);
+        const results = {};
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                results[cursor.value.sessionId] = {
+                    session: cursor.value.session,
+                    lastReceivedMessageTs: cursor.value.lastReceivedMessageTs,
+                };
+                cursor.continue();
+            } else {
+                try {
+                    func(results);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.get([deviceKey, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func({
+                        session: getReq.result.session,
+                        lastReceivedMessageTs: getReq.result.lastReceivedMessageTs,
+                    });
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    getAllEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                func(cursor.value);
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
+        const objectStore = txn.objectStore("sessions");
+        objectStore.put({
+            deviceKey,
+            sessionId,
+            session: sessionInfo.session,
+            lastReceivedMessageTs: sessionInfo.lastReceivedMessageTs,
+        });
+    }
+
+    // Inbound group sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.get([senderCurve25519Key, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                try {
+                    func({
+                        senderKey: cursor.value.senderCurve25519Key,
+                        sessionId: cursor.value.sessionId,
+                        sessionData: cursor.value.session,
+                    });
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const addReq = objectStore.add({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+        addReq.onerror = (ev) => {
+            if (addReq.error.name === 'ConstraintError') {
+                // This stops the error from triggering the txn's onerror
+                ev.stopPropagation();
+                // ...and this stops it from aborting the transaction
+                ev.preventDefault();
+                logger.log(
+                    "Ignoring duplicate inbound group session: " +
+                    senderCurve25519Key + " / " + sessionId,
+                );
+            } else {
+                abortWithException(txn, new Error(
+                    "Failed to add inbound group session: " + addReq.error,
+                ));
+            }
+        };
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        objectStore.put({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        const objectStore = txn.objectStore("device_data");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        const objectStore = txn.objectStore("device_data");
+        objectStore.put(deviceData, "-");
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        const objectStore = txn.objectStore("rooms");
+        objectStore.put(roomInfo, roomId);
+    }
+
+    getEndToEndRooms(txn, func) {
+        const rooms = {};
+        const objectStore = txn.objectStore("rooms");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                rooms[cursor.key] = cursor.value;
+                cursor.continue();
+            } else {
+                try {
+                    func(rooms);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    // session backups
+
+    getSessionsNeedingBackup(limit) {
+        return new Promise((resolve, reject) => {
+            const sessions = [];
+
+            const txn = this._db.transaction(
+                ["sessions_needing_backup", "inbound_group_sessions"],
+                "readonly",
+            );
+            txn.onerror = reject;
+            txn.oncomplete = function() {
+                resolve(sessions);
+            };
+            const objectStore = txn.objectStore("sessions_needing_backup");
+            const sessionStore = txn.objectStore("inbound_group_sessions");
+            const getReq = objectStore.openCursor();
+            getReq.onsuccess = function() {
+                const cursor = getReq.result;
+                if (cursor) {
+                    const sessionGetReq = sessionStore.get(cursor.key);
+                    sessionGetReq.onsuccess = function() {
+                        sessions.push({
+                            senderKey: sessionGetReq.result.senderCurve25519Key,
+                            sessionId: sessionGetReq.result.sessionId,
+                            sessionData: sessionGetReq.result.session,
+                        });
+                    };
+                    if (!limit || sessions.length < limit) {
+                        cursor.continue();
+                    }
+                }
+            };
+        });
+    }
+
+    countSessionsNeedingBackup(txn) {
+        if (!txn) {
+            txn = this._db.transaction("sessions_needing_backup", "readonly");
+        }
+        const objectStore = txn.objectStore("sessions_needing_backup");
+        return new Promise((resolve, reject) => {
+            const req = objectStore.count();
+            req.onerror = reject;
+            req.onsuccess = () => resolve(req.result);
+        });
+    }
+
+    unmarkSessionsNeedingBackup(sessions, txn) {
+        if (!txn) {
+            txn = this._db.transaction("sessions_needing_backup", "readwrite");
+        }
+        const objectStore = txn.objectStore("sessions_needing_backup");
+        return Promise.all(sessions.map((session) => {
+            return new Promise((resolve, reject) => {
+                const req = objectStore.delete([session.senderKey, session.sessionId]);
+                req.onsuccess = resolve;
+                req.onerror = reject;
+            });
+        }));
+    }
+
+    markSessionsNeedingBackup(sessions, txn) {
+        if (!txn) {
+            txn = this._db.transaction("sessions_needing_backup", "readwrite");
+        }
+        const objectStore = txn.objectStore("sessions_needing_backup");
+        return Promise.all(sessions.map((session) => {
+            return new Promise((resolve, reject) => {
+                const req = objectStore.put({
+                    senderCurve25519Key: session.senderKey,
+                    sessionId: session.sessionId,
+                });
+                req.onsuccess = resolve;
+                req.onerror = reject;
+            });
+        }));
+    }
+
+    doTxn(mode, stores, func) {
+        const txn = this._db.transaction(stores, mode);
+        const promise = promiseifyTxn(txn);
+        const result = func(txn);
+        return promise.then(() => {
+            return result;
+        });
+    }
+}
+
+export function upgradeDatabase(db, oldVersion) {
+    logger.log(
+        `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
+            + ` to ${VERSION}`,
+    );
+    if (oldVersion < 1) { // The database did not previously exist.
+        createDatabase(db);
+    }
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", {
+            keyPath: ["deviceKey", "sessionId"],
+        });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
+    if (oldVersion < 7) {
+        db.createObjectStore("sessions_needing_backup", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
+    // Expand as needed.
+}
+
+function createDatabase(db) {
+    const outgoingRoomKeyRequestsStore =
+        db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
+
+    // we assume that the RoomKeyRequestBody will have room_id and session_id
+    // properties, to make the index efficient.
+    outgoingRoomKeyRequestsStore.createIndex("session",
+        ["requestBody.room_id", "requestBody.session_id"],
+    );
+
+    outgoingRoomKeyRequestsStore.createIndex("state", "state");
+}
+
+/*
+ * Aborts a transaction with a given exception
+ * The transaction promise will be rejected with this exception.
+ */
+function abortWithException(txn, e) {
+    // We cheekily stick our exception onto the transaction object here
+    // We could alternatively make the thing we pass back to the app
+    // an object containing the transaction and exception.
+    txn._mx_abortexception = e;
+    try {
+        txn.abort();
+    } catch (e) {
+        // sometimes we won't be able to abort the transaction
+        // (ie. if it's aborted or completed)
+    }
+}
+
+function promiseifyTxn(txn) {
+    return new Promise((resolve, reject) => {
+        txn.oncomplete = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            resolve();
+        };
+        txn.onerror = (event) => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            } else {
+                reject(event.target.error);
+            }
+        };
+        txn.onabort = () => reject(txn._mx_abortexception);
+    });
+}
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from 'q';
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import LocalStorageCryptoStore from './localStorage-crypto-store';
+import MemoryCryptoStore from './memory-crypto-store';
+import * as IndexedDBCryptoStoreBackend from './indexeddb-crypto-store-backend';
+import {InvalidCryptoStoreError} from '../../errors';

 /**
 * Internal module. indexeddb storage for e2e.
@@ -22,9 +29,10 @@ import q from 'q';
 * @module
 */

-const VERSION = 1;
-
 /**
+ * An implementation of CryptoStore, which is normally backed by an indexeddb,
+ * but with fallback to MemoryCryptoStore.
+ *
 * @implements {module:crypto/store/base~CryptoStore}
 */
 export default class IndexedDBCryptoStore {
@@ -35,68 +43,93 @@ export default class IndexedDBCryptoStore {
     * @param {string} dbName   name of db to connect to
     */
    constructor(indexedDB, dbName) {
-        if (!indexedDB) {
-            throw new Error("must pass indexedDB into IndexedDBCryptoStore");
-        }
        this._indexedDB = indexedDB;
        this._dbName = dbName;
-        this._dbPromise = null;
+        this._backendPromise = null;
    }

    /**
-     * Ensure the database exists and is up-to-date
+     * Ensure the database exists and is up-to-date, or fall back to
+     * a local storage or in-memory store.
     *
-     * @return {Promise} resolves to an instance of IDBDatabase when
-     * the database is ready
+     * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
+     * or a MemoryCryptoStore
     */
-    connect() {
-        if (this._dbPromise) {
-            return this._dbPromise;
+    _connect() {
+        if (this._backendPromise) {
+            return this._backendPromise;
        }

-        this._dbPromise = new q.Promise((resolve, reject) => {
-            console.log(`connecting to indexeddb ${this._dbName}`);
-            const req = this._indexedDB.open(this._dbName, VERSION);
+        this._backendPromise = new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            logger.log(`connecting to indexeddb ${this._dbName}`);
+
+            const req = this._indexedDB.open(
+                this._dbName, IndexedDBCryptoStoreBackend.VERSION,
+            );

            req.onupgradeneeded = (ev) => {
                const db = ev.target.result;
                const oldVersion = ev.oldVersion;
-                console.log(
-                    `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
-                    + ` to ${VERSION}`,
-                );
-                if (oldVersion < 1) { // The database did not previously exist.
-                    createDatabase(db);
-                }
-                // Expand as needed.
+                IndexedDBCryptoStoreBackend.upgradeDatabase(db, oldVersion);
            };

            req.onblocked = () => {
-                reject(new Error(
-                    "unable to upgrade indexeddb because it is open elsewhere",
-                ));
+                logger.log(
+                    `can't yet open IndexedDBCryptoStore because it is open elsewhere`,
+                );
            };

            req.onerror = (ev) => {
-                reject(new Error(
-                    "unable to connect to indexeddb: " + ev.target.error,
-                ));
+                reject(ev.target.error);
            };

            req.onsuccess = (r) => {
                const db = r.target.result;

-                // make sure we close the db on `onversionchange` - otherwise
-                // attempts to delete the database will block (and subsequent
-                // attempts to re-create it will also block).
-                db.onversionchange = (ev) => {
-                    db.close();
-                };
-
-                resolve(db);
+                logger.log(`connected to indexeddb ${this._dbName}`);
+                resolve(new IndexedDBCryptoStoreBackend.Backend(db));
            };
+        }).then((backend) => {
+            // Edge has IndexedDB but doesn't support compund keys which we use fairly extensively.
+            // Try a dummy query which will fail if the browser doesn't support compund keys, so
+            // we can fall back to a different backend.
+            return backend.doTxn(
+                'readonly',
+                [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                (txn) => {
+                    backend.getEndToEndInboundGroupSession('', '', txn, () => {});
+                }).then(() => {
+                    return backend;
+                },
+            );
+        }).catch((e) => {
+            if (e.name === 'VersionError') {
+                logger.warn("Crypto DB is too new for us to use!", e);
+                // don't fall back to a different store: the user has crypto data
+                // in this db so we should use it or nothing at all.
+                throw new InvalidCryptoStoreError(InvalidCryptoStoreError.TOO_NEW);
+            }
+            logger.warn(
+                `unable to connect to indexeddb ${this._dbName}` +
+                    `: falling back to localStorage store: ${e}`,
+            );
+
+            try {
+                return new LocalStorageCryptoStore(global.localStorage);
+            } catch (e) {
+                logger.warn(
+                    `unable to open localStorage: falling back to in-memory store: ${e}`,
+                );
+                return new MemoryCryptoStore();
+            }
        });
-        return this._dbPromise;
+
+        return this._backendPromise;
    }

    /**
@@ -105,39 +138,414 @@ export default class IndexedDBCryptoStore {
     * @returns {Promise} resolves when the store has been cleared.
     */
    deleteAllData() {
-        return new q.Promise((resolve, reject) => {
-            console.log(`Removing indexeddb instance: ${this._dbName}`);
+        return new Promise((resolve, reject) => {
+            if (!this._indexedDB) {
+                reject(new Error('no indexeddb support available'));
+                return;
+            }
+
+            logger.log(`Removing indexeddb instance: ${this._dbName}`);
            const req = this._indexedDB.deleteDatabase(this._dbName);

            req.onblocked = () => {
-                reject(new Error(
-                    "unable to delete indexeddb because it is open elsewhere",
-                ));
+                logger.log(
+                    `can't yet delete IndexedDBCryptoStore because it is open elsewhere`,
+                );
            };

            req.onerror = (ev) => {
-                reject(new Error(
-                    "unable to delete indexeddb: " + ev.target.error,
-                ));
+                reject(ev.target.error);
            };

            req.onsuccess = () => {
-                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                logger.log(`Removed indexeddb instance: ${this._dbName}`);
                resolve();
            };
+        }).catch((e) => {
+            // in firefox, with indexedDB disabled, this fails with a
+            // DOMError. We treat this as non-fatal, so that people can
+            // still use the app.
+            logger.warn(`unable to delete IndexedDBCryptoStore: ${e}`);
+        });
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        return this._connect().then((backend) => {
+            return backend.getOrAddOutgoingRoomKeyRequest(request);
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequest(requestBody);
+        });
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states. If there are multiple
+     *    requests in those states, an arbitrary one is chosen.
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequestByState(wantedStates);
+        });
+    }
+
+    /**
+     * Look for room key requests by target device and state
+     *
+     * @param {string} userId Target user ID
+     * @param {string} deviceId Target device ID
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to a list of all the
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     */
+    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
+        return this._connect().then((backend) => {
+            return backend.getOutgoingRoomKeyRequestsByTarget(
+                userId, deviceId, wantedStates,
+            );
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        return this._connect().then((backend) => {
+            return backend.updateOutgoingRoomKeyRequest(
+                requestId, expectedState, updates,
+            );
+        });
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        return this._connect().then((backend) => {
+            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
+        });
+    }
+
+    // Olm Account
+
+    /*
+     * Get the account pickle from the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the account pickle
+     */
+    getAccount(txn, func) {
+        this._backendPromise.value().getAccount(txn, func);
+    }
+
+    /*
+     * Write the account pickle to the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} newData The new account pickle to store.
+     */
+    storeAccount(txn, newData) {
+        this._backendPromise.value().storeAccount(txn, newData);
+    }
+
+    // Olm sessions
+
+    /**
+     * Returns the number of end-to-end sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(int)} func Called with the count of sessions
+     */
+    countEndToEndSessions(txn, func) {
+        this._backendPromise.value().countEndToEndSessions(txn, func);
+    }
+
+    /**
+     * Retrieve a specific end-to-end session between the logged-in user
+     * and another device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID of the session to retrieve
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to session information object with 'session' key being the
+     *     Base64 end-to-end session and lastReceivedMessageTs being the
+     *     timestamp in milliseconds at which the session last received
+     *     a message.
+     */
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndSession(deviceKey, sessionId, txn, func);
+    }
+
+    /**
+     * Retrieve the end-to-end sessions between the logged-in user and another
+     * device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to session information object with 'session' key being the
+     *     Base64 end-to-end session and lastReceivedMessageTs being the
+     *     timestamp in milliseconds at which the session last received
+     *     a message.
+     */
+    getEndToEndSessions(deviceKey, txn, func) {
+        this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func);
+    }
+
+    /**
+     * Retrieve all end-to-end sessions
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called one for each session with
+     *     an object with, deviceKey, lastReceivedMessageTs, sessionId
+     *     and session keys.
+     */
+    getAllEndToEndSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndSessions(txn, func);
+    }
+
+    /**
+     * Store a session between the logged-in user and another device
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID for this end-to-end session.
+     * @param {string} sessionInfo Session information object
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
+        this._backendPromise.value().storeEndToEndSession(
+            deviceKey, sessionId, sessionInfo, txn,
+        );
+    }
+
+    // Inbound group saessions
+
+    /**
+     * Retrieve the end-to-end inbound group session for a given
+     * server key and session ID
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
+    }
+
+    /**
+     * Fetches all inbound group sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called once for each group session
+     *     in the store with an object having keys {senderKey, sessionId,
+     *     sessionData}, then once with null to indicate the end of the list.
+     */
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndInboundGroupSessions(txn, func);
+    }
+
+    /**
+     * Adds an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, the session will not be added.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    /**
+     * Writes an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, it will be overwritten.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    // End-to-end device tracking
+
+    /**
+     * Store the state of all tracked devices
+     * This contains devices for each user, a tracking state for each user
+     * and a sync token matching the point in time the snapshot represents.
+     * These all need to be written out in full each time such that the snapshot
+     * is always consistent, so they are stored in one object.
+     *
+     * @param {Object} deviceData
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._backendPromise.value().storeEndToEndDeviceData(deviceData, txn);
+    }
+
+    /**
+     * Get the state of all tracked devices
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the
+     *     device data
+     */
+    getEndToEndDeviceData(txn, func) {
+        this._backendPromise.value().getEndToEndDeviceData(txn, func);
+    }
+
+    // End to End Rooms
+
+    /**
+     * Store the end-to-end state for a room.
+     * @param {string} roomId The room's ID.
+     * @param {object} roomInfo The end-to-end info for the room.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._backendPromise.value().storeEndToEndRoom(roomId, roomInfo, txn);
+    }
+
+    /**
+     * Get an object of roomId->roomInfo for all e2e rooms in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the end to end encrypted rooms
+     */
+    getEndToEndRooms(txn, func) {
+        this._backendPromise.value().getEndToEndRooms(txn, func);
+    }
+
+    // session backups
+
+    /**
+     * Get the inbound group sessions that need to be backed up.
+     * @param {integer} limit The maximum number of sessions to retrieve.  0
+     * for no limit.
+     * @returns {Promise} resolves to an array of inbound group sessions
+     */
+    getSessionsNeedingBackup(limit) {
+        return this._connect().then((backend) => {
+            return backend.getSessionsNeedingBackup(limit);
+        });
+    }
+
+    /**
+     * Count the inbound group sessions that need to be backed up.
+     * @param {*} txn An active transaction. See doTxn(). (optional)
+     * @returns {Promise} resolves to the number of sessions
+     */
+    countSessionsNeedingBackup(txn) {
+        return this._connect().then((backend) => {
+            return backend.countSessionsNeedingBackup(txn);
+        });
+    }
+
+    /**
+     * Unmark sessions as needing to be backed up.
+     * @param {Array<object>} sessions The sessions that need to be backed up.
+     * @param {*} txn An active transaction. See doTxn(). (optional)
+     * @returns {Promise} resolves when the sessions are unmarked
+     */
+    unmarkSessionsNeedingBackup(sessions, txn) {
+        return this._connect().then((backend) => {
+            return backend.unmarkSessionsNeedingBackup(sessions, txn);
+        });
+    }
+
+    /**
+     * Mark sessions as needing to be backed up.
+     * @param {Array<object>} sessions The sessions that need to be backed up.
+     * @param {*} txn An active transaction. See doTxn(). (optional)
+     * @returns {Promise} resolves when the sessions are marked
+     */
+    markSessionsNeedingBackup(sessions, txn) {
+        return this._connect().then((backend) => {
+            return backend.markSessionsNeedingBackup(sessions, txn);
+        });
+    }
+
+    /**
+     * Perform a transaction on the crypto store. Any store methods
+     * that require a transaction (txn) object to be passed in may
+     * only be called within a callback of either this function or
+     * one of the store functions operating on the same transaction.
+     *
+     * @param {string} mode 'readwrite' if you need to call setter
+     *     functions with this transaction. Otherwise, 'readonly'.
+     * @param {string[]} stores List IndexedDBCryptoStore.STORE_*
+     *     options representing all types of object that will be
+     *     accessed or written to with this transaction.
+     * @param {function(*)} func Function called with the
+     *     transaction object: an opaque object that should be passed
+     *     to store functions.
+     * @return {Promise} Promise that resolves with the result of the `func`
+     *     when the transaction is complete. If the backend is
+     *     async (ie. the indexeddb backend) any of the callback
+     *     functions throwing an exception will cause this promise to
+     *     reject with that exception. On synchronous backends, the
+     *     exception will propagate to the caller of the getFoo method.
+     */
+    doTxn(mode, stores, func) {
+        return this._connect().then((backend) => {
+            return backend.doTxn(mode, stores, func);
        });
    }
 }

-function createDatabase(db) {
-    const outgoingRoomKeyRequestsStore =
-        db.createObjectStore("outgoingRoomKeyRequests", { keyPath: "requestId" });
-
-    // we assume that the RoomKeyRequestBody will have room_id and session_id
-    // properties, to make the index efficient.
-    outgoingRoomKeyRequestsStore.createIndex("session",
-        ["requestBody.room_id", "requestBody.session_id"],
-    );
-
-    outgoingRoomKeyRequestsStore.createIndex("state", "state");
-}
+IndexedDBCryptoStore.STORE_ACCOUNT = 'account';
+IndexedDBCryptoStore.STORE_SESSIONS = 'sessions';
+IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS = 'inbound_group_sessions';
+IndexedDBCryptoStore.STORE_DEVICE_DATA = 'device_data';
+IndexedDBCryptoStore.STORE_ROOMS = 'rooms';
+IndexedDBCryptoStore.STORE_BACKUP = 'sessions_needing_backup';
@@ -0,0 +1,296 @@
+/*
+Copyright 2017, 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import MemoryCryptoStore from './memory-crypto-store.js';
+
+/**
+ * Internal module. Partial localStorage backed storage for e2e.
+ * This is not a full crypto store, just the in-memory store with
+ * some things backed by localStorage. It exists because indexedDB
+ * is broken in Firefox private mode or set to, "will not remember
+ * history".
+ *
+ * @module
+ */
+
+const E2E_PREFIX = "crypto.";
+const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
+const KEY_DEVICE_DATA = E2E_PREFIX + "device_data";
+const KEY_INBOUND_SESSION_PREFIX = E2E_PREFIX + "inboundgroupsessions/";
+const KEY_ROOMS_PREFIX = E2E_PREFIX + "rooms/";
+const KEY_SESSIONS_NEEDING_BACKUP = E2E_PREFIX + "sessionsneedingbackup";
+
+function keyEndToEndSessions(deviceKey) {
+    return E2E_PREFIX + "sessions/" + deviceKey;
+}
+
+function keyEndToEndInboundGroupSession(senderKey, sessionId) {
+    return KEY_INBOUND_SESSION_PREFIX + senderKey + "/" + sessionId;
+}
+
+function keyEndToEndRoomsPrefix(roomId) {
+    return KEY_ROOMS_PREFIX + roomId;
+}
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class LocalStorageCryptoStore extends MemoryCryptoStore {
+    constructor(webStore) {
+        super();
+        this.store = webStore;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        let count = 0;
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) ++count;
+        }
+        func(count);
+    }
+
+    _getEndToEndSessions(deviceKey, txn, func) {
+        const sessions = getJsonItem(this.store, keyEndToEndSessions(deviceKey));
+        const fixedSessions = {};
+
+        // fix up any old sessions to be objects rather than just the base64 pickle
+        for (const [sid, val] of Object.entries(sessions || {})) {
+            if (typeof val === 'string') {
+                fixedSessions[sid] = {
+                    session: val,
+                };
+            } else {
+                fixedSessions[sid] = val;
+            }
+        }
+
+        return fixedSessions;
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const sessions = this._getEndToEndSessions(deviceKey);
+        func(sessions[sessionId] || {});
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._getEndToEndSessions(deviceKey) || {});
+    }
+
+    getAllEndToEndSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) {
+                const deviceKey = this.store.key(i).split('/')[1];
+                for (const sess of Object.values(this._getEndToEndSessions(deviceKey))) {
+                    func(sess);
+                }
+            }
+        }
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
+        const sessions = this._getEndToEndSessions(deviceKey) || {};
+        sessions[sessionId] = sessionInfo;
+        setJsonItem(
+            this.store, keyEndToEndSessions(deviceKey), sessions,
+        );
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        ));
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                // we can't use split, as the components we are trying to split out
+                // might themselves contain '/' characters. We rely on the
+                // senderKey being a (32-byte) curve25519 key, base64-encoded
+                // (hence 43 characters long).
+
+                func({
+                    senderKey: key.substr(KEY_INBOUND_SESSION_PREFIX.length, 43),
+                    sessionId: key.substr(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    sessionData: getJsonItem(this.store, key),
+                });
+            }
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const existing = getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        );
+        if (!existing) {
+            this.storeEndToEndInboundGroupSession(
+                senderCurve25519Key, sessionId, sessionData, txn,
+            );
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        setJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+            sessionData,
+        );
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        func(getJsonItem(
+            this.store, KEY_DEVICE_DATA,
+        ));
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        setJsonItem(
+            this.store, KEY_DEVICE_DATA, deviceData,
+        );
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        setJsonItem(
+            this.store, keyEndToEndRoomsPrefix(roomId), roomInfo,
+        );
+    }
+
+    getEndToEndRooms(txn, func) {
+        const result = {};
+        const prefix = keyEndToEndRoomsPrefix('');
+
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(prefix)) {
+                const roomId = key.substr(prefix.length);
+                result[roomId] = getJsonItem(this.store, key);
+            }
+        }
+        func(result);
+    }
+
+    getSessionsNeedingBackup(limit) {
+        const sessionsNeedingBackup
+              = getJsonItem(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        const sessions = [];
+
+        for (const session in sessionsNeedingBackup) {
+            if (Object.prototype.hasOwnProperty.call(sessionsNeedingBackup, session)) {
+                // see getAllEndToEndInboundGroupSessions for the magic number explanations
+                const senderKey = session.substr(0, 43);
+                const sessionId = session.substr(44);
+                this.getEndToEndInboundGroupSession(
+                    senderKey, sessionId, null,
+                    (sessionData) => {
+                        sessions.push({
+                            senderKey: senderKey,
+                            sessionId: sessionId,
+                            sessionData: sessionData,
+                        });
+                    },
+                );
+                if (limit && session.length >= limit) {
+                    break;
+                }
+            }
+        }
+        return Promise.resolve(sessions);
+    }
+
+    countSessionsNeedingBackup() {
+        const sessionsNeedingBackup
+              = getJsonItem(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        return Promise.resolve(Object.keys(sessionsNeedingBackup).length);
+    }
+
+    unmarkSessionsNeedingBackup(sessions) {
+        const sessionsNeedingBackup
+              = getJsonItem(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        for (const session of sessions) {
+            delete sessionsNeedingBackup[session.senderKey + '/' + session.sessionId];
+        }
+        setJsonItem(
+            this.store, KEY_SESSIONS_NEEDING_BACKUP, sessionsNeedingBackup,
+        );
+        return Promise.resolve();
+    }
+
+    markSessionsNeedingBackup(sessions) {
+        const sessionsNeedingBackup
+              = getJsonItem(this.store, KEY_SESSIONS_NEEDING_BACKUP) || {};
+        for (const session of sessions) {
+            sessionsNeedingBackup[session.senderKey + '/' + session.sessionId] = true;
+        }
+        setJsonItem(
+            this.store, KEY_SESSIONS_NEEDING_BACKUP, sessionsNeedingBackup,
+        );
+        return Promise.resolve();
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
+        return Promise.resolve();
+    }
+
+    // Olm account
+
+    getAccount(txn, func) {
+        const account = getJsonItem(this.store, KEY_END_TO_END_ACCOUNT);
+        func(account);
+    }
+
+    storeAccount(txn, newData) {
+        setJsonItem(
+            this.store, KEY_END_TO_END_ACCOUNT, newData,
+        );
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
+}
+
+function getJsonItem(store, key) {
+    try {
+        // if the key is absent, store.getItem() returns null, and
+        // JSON.parse(null) === null, so this returns null.
+        return JSON.parse(store.getItem(key));
+    } catch (e) {
+        logger.log("Error: Failed to get key %s: %s", key, e.stack || e);
+        logger.log(e.stack);
+    }
+    return null;
+}
+
+function setJsonItem(store, key, val) {
+    store.setItem(key, JSON.stringify(val));
+}
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from 'q';
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import utils from '../../utils';

 /**
 * Internal module. in-memory storage for e2e.
@@ -27,6 +31,19 @@ import q from 'q';
 */
 export default class MemoryCryptoStore {
    constructor() {
+        this._outgoingRoomKeyRequests = [];
+        this._account = null;
+
+        // Map of {devicekey -> {sessionId -> session pickle}}
+        this._sessions = {};
+        // Map of {senderCurve25519Key+'/'+sessionId -> session data object}
+        this._inboundGroupSessions = {};
+        // Opaque device data object
+        this._deviceData = null;
+        // roomId -> Opaque roomInfo object
+        this._rooms = {};
+        // Set of {senderCurve25519Key+'/'+sessionId}
+        this._sessionsNeedingBackup = {};
    }

    /**
@@ -35,6 +52,313 @@ export default class MemoryCryptoStore {
     * @returns {Promise} Promise which resolves when the store has been cleared.
     */
    deleteAllData() {
-        return q();
+        return Promise.resolve();
+    }
+
+    /**
+     * Look for an existing outgoing room key request, and if none is found,
+     * add a new one
+     *
+     * @param {module:crypto/store/base~OutgoingRoomKeyRequest} request
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}: either the
+     *    same instance as passed in, or the existing one.
+     */
+    getOrAddOutgoingRoomKeyRequest(request) {
+        const requestBody = request.requestBody;
+
+        return Promise.try(() => {
+            // first see if we already have an entry for this request.
+            const existing = this._getOutgoingRoomKeyRequest(requestBody);
+
+            if (existing) {
+                // this entry matches the request - return it.
+                logger.log(
+                    `already have key request outstanding for ` +
+                    `${requestBody.room_id} / ${requestBody.session_id}: ` +
+                    `not sending another`,
+                );
+                return existing;
+            }
+
+            // we got to the end of the list without finding a match
+            // - add the new request.
+            logger.log(
+                `enqueueing key request for ${requestBody.room_id} / ` +
+                requestBody.session_id,
+            );
+            this._outgoingRoomKeyRequests.push(request);
+            return request;
+        });
+    }
+
+    /**
+     * Look for an existing room key request
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {Promise} resolves to the matching
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    not found
+     */
+    getOutgoingRoomKeyRequest(requestBody) {
+        return Promise.resolve(this._getOutgoingRoomKeyRequest(requestBody));
+    }
+
+    /**
+     * Looks for existing room key request, and returns the result synchronously.
+     *
+     * @internal
+     *
+     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     *    existing request to look for
+     *
+     * @return {module:crypto/store/base~OutgoingRoomKeyRequest?}
+     *    the matching request, or null if not found
+     */
+    _getOutgoingRoomKeyRequest(requestBody) {
+        for (const existing of this._outgoingRoomKeyRequests) {
+            if (utils.deepCompare(existing.requestBody, requestBody)) {
+                return existing;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Look for room key requests by state
+     *
+     * @param {Array<Number>} wantedStates list of acceptable states
+     *
+     * @return {Promise} resolves to the a
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}, or null if
+     *    there are no pending requests in those states
+     */
+    getOutgoingRoomKeyRequestByState(wantedStates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            for (const state of wantedStates) {
+                if (req.state === state) {
+                    return Promise.resolve(req);
+                }
+            }
+        }
+        return Promise.resolve(null);
+    }
+
+    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
+        const results = [];
+
+        for (const req of this._outgoingRoomKeyRequests) {
+            for (const state of wantedStates) {
+                if (req.state === state && req.recipients.includes({userId, deviceId})) {
+                    results.push(req);
+                }
+            }
+        }
+        return Promise.resolve(results);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and update it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     * @param {Object} updates        name/value map of updates to apply
+     *
+     * @returns {Promise} resolves to
+     *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
+     *    updated request, or null if no matching row was found
+     */
+    updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
+        for (const req of this._outgoingRoomKeyRequests) {
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                logger.warn(
+                    `Cannot update room key request from ${expectedState} ` +
+                    `as it was already updated to ${req.state}`,
+                );
+                return Promise.resolve(null);
+            }
+            Object.assign(req, updates);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+
+    /**
+     * Look for an existing room key request by id and state, and delete it if
+     * found
+     *
+     * @param {string} requestId      ID of request to update
+     * @param {number} expectedState  state we expect to find the request in
+     *
+     * @returns {Promise} resolves once the operation is completed
+     */
+    deleteOutgoingRoomKeyRequest(requestId, expectedState) {
+        for (let i = 0; i < this._outgoingRoomKeyRequests.length; i++) {
+            const req = this._outgoingRoomKeyRequests[i];
+
+            if (req.requestId !== requestId) {
+                continue;
+            }
+
+            if (req.state != expectedState) {
+                logger.warn(
+                    `Cannot delete room key request in state ${req.state} `
+                    + `(expected ${expectedState})`,
+                );
+                return Promise.resolve(null);
+            }
+
+            this._outgoingRoomKeyRequests.splice(i, 1);
+            return Promise.resolve(req);
+        }
+
+        return Promise.resolve(null);
+    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        func(this._account);
+    }
+
+    storeAccount(txn, newData) {
+        this._account = newData;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        return Object.keys(this._sessions).length;
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const deviceSessions = this._sessions[deviceKey] || {};
+        func(deviceSessions[sessionId] || null);
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._sessions[deviceKey] || {});
+    }
+
+    getAllEndToEndSessions(txn, func) {
+        for (const deviceSessions of Object.values(this._sessions)) {
+            for (const sess of Object.values(deviceSessions)) {
+                func(sess);
+            }
+        }
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
+        let deviceSessions = this._sessions[deviceKey];
+        if (deviceSessions === undefined) {
+            deviceSessions = {};
+            this._sessions[deviceKey] = deviceSessions;
+        }
+        deviceSessions[sessionId] = sessionInfo;
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] || null);
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (const key of Object.keys(this._inboundGroupSessions)) {
+            // we can't use split, as the components we are trying to split out
+            // might themselves contain '/' characters. We rely on the
+            // senderKey being a (32-byte) curve25519 key, base64-encoded
+            // (hence 43 characters long).
+
+            func({
+                senderKey: key.substr(0, 43),
+                sessionId: key.substr(44),
+                sessionData: this._inboundGroupSessions[key],
+            });
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const k = senderCurve25519Key+'/'+sessionId;
+        if (this._inboundGroupSessions[k] === undefined) {
+            this._inboundGroupSessions[k] = sessionData;
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] = sessionData;
+    }
+
+    // Device Data
+
+    getEndToEndDeviceData(txn, func) {
+        func(this._deviceData);
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._deviceData = deviceData;
+    }
+
+    // E2E rooms
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._rooms[roomId] = roomInfo;
+    }
+
+    getEndToEndRooms(txn, func) {
+        func(this._rooms);
+    }
+
+    getSessionsNeedingBackup(limit) {
+        const sessions = [];
+        for (const session in this._sessionsNeedingBackup) {
+            if (this._inboundGroupSessions[session]) {
+                sessions.push({
+                    senderKey: session.substr(0, 43),
+                    sessionId: session.substr(44),
+                    sessionData: this._inboundGroupSessions[session],
+                });
+                if (limit && session.length >= limit) {
+                    break;
+                }
+            }
+        }
+        return Promise.resolve(sessions);
+    }
+
+    countSessionsNeedingBackup() {
+        return Promise.resolve(Object.keys(this._sessionsNeedingBackup).length);
+    }
+
+    unmarkSessionsNeedingBackup(sessions) {
+        for (const session of sessions) {
+            const sessionKey = session.senderKey + '/' + session.sessionId;
+            delete this._sessionsNeedingBackup[sessionKey];
+        }
+        return Promise.resolve();
+    }
+
+    markSessionsNeedingBackup(sessions) {
+        for (const session of sessions) {
+            const sessionKey = session.senderKey + '/' + session.sessionId;
+            this._sessionsNeedingBackup[sessionKey] = true;
+        }
+        return Promise.resolve();
+    }
+
+    // Session key backups
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
    }
 }
@@ -0,0 +1,200 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Base class for verification methods.
+ * @module crypto/verification/Base
+ */
+
+import {MatrixEvent} from '../../models/event';
+import {EventEmitter} from 'events';
+
+export default class VerificationBase extends EventEmitter {
+    /**
+     * Base class for verification methods.
+     *
+     * <p>Once a verifier object is created, the verification can be started by
+     * calling the verify() method, which will return a promise that will
+     * resolve when the verification is completed, or reject if it could not
+     * complete.</p>
+     *
+     * <p>Subclasses must have a NAME class property.</p>
+     *
+     * @class
+     *
+     * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
+     *
+     * @param {string} userId the user ID that is being verified
+     *
+     * @param {string} deviceId the device ID that is being verified
+     *
+     * @param {string} transactionId the transaction ID to be used when sending events
+     *
+     * @param {object} startEvent the m.key.verification.start event that
+     * initiated this verification, if any
+     *
+     * @param {object} request the key verification request object related to
+     * this verification, if any
+     *
+     * @param {object} parent parent verification for this verification, if any
+     */
+    constructor(baseApis, userId, deviceId, transactionId, startEvent, request, parent) {
+        super();
+        this._baseApis = baseApis;
+        this.userId = userId;
+        this.deviceId = deviceId;
+        this.transactionId = transactionId;
+        this.startEvent = startEvent;
+        this.request = request;
+        this._parent = parent;
+        this._done = false;
+        this._promise = null;
+    }
+
+    _sendToDevice(type, content) {
+        if (this._done) {
+            return Promise.reject(new Error("Verification is already done"));
+        }
+        content.transaction_id = this.transactionId;
+        return this._baseApis.sendToDevice(type, {
+            [this.userId]: { [this.deviceId]: content },
+        });
+    }
+
+    _waitForEvent(type) {
+        if (this._done) {
+            return Promise.reject(new Error("Verification is already done"));
+        }
+        this._expectedEvent = type;
+        return new Promise((resolve, reject) => {
+            this._resolveEvent = resolve;
+            this._rejectEvent = reject;
+        });
+    }
+
+    handleEvent(e) {
+        if (this._done) {
+            return;
+        } else if (e.getType() === this._expectedEvent) {
+            this._expectedEvent = undefined;
+            this._rejectEvent = undefined;
+            this._resolveEvent(e);
+        } else {
+            this._expectedEvent = undefined;
+            const exception = new Error(
+                "Unexpected message: expecting " + this._expectedEvent
+                    + " but got " + e.getType(),
+            );
+            if (this._rejectEvent) {
+                const reject = this._rejectEvent;
+                this._rejectEvent = undefined;
+                reject(exception);
+            }
+            this.cancel(exception);
+        }
+    }
+
+    done() {
+        if (!this._done) {
+            this._resolve();
+        }
+    }
+
+    cancel(e) {
+        if (!this._done) {
+            if (this.userId && this.deviceId && this.transactionId) {
+                // send a cancellation to the other user (if it wasn't
+                // cancelled by the other user)
+                if (e instanceof MatrixEvent) {
+                    const sender = e.getSender();
+                    if (sender !== this.userId) {
+                        const content = e.getContent();
+                        if (e.getType() === "m.key.verification.cancel") {
+                            content.code = content.code || "m.unknown";
+                            content.reason = content.reason || content.body
+                                || "Unknown reason";
+                            content.transaction_id = this.transactionId;
+                            this._sendToDevice("m.key.verification.cancel", content);
+                        } else {
+                            this._sendToDevice("m.key.verification.cancel", {
+                                code: "m.unknown",
+                                reason: content.body || "Unknown reason",
+                                transaction_id: this.transactionId,
+                            });
+                        }
+                    }
+                } else {
+                    this._sendToDevice("m.key.verification.cancel", {
+                        code: "m.unknown",
+                        reason: e.toString(),
+                        transaction_id: this.transactionId,
+                    });
+                }
+            }
+            if (this._promise !== null) {
+                this._reject(e);
+            } else {
+                this._promise = Promise.reject(e);
+            }
+            // Also emit a 'cancel' event that the app can listen for to detect cancellation
+            // before calling verify()
+            this.emit('cancel', e);
+        }
+    }
+
+    /**
+     * Begin the key verification
+     *
+     * @returns {Promise} Promise which resolves when the verification has
+     *     completed.
+     */
+    verify() {
+        if (this._promise) return this._promise;
+
+        this._promise = new Promise((resolve, reject) => {
+            this._resolve = (...args) => {
+                this._done = true;
+                resolve(...args);
+            };
+            this._reject = (...args) => {
+                this._done = true;
+                reject(...args);
+            };
+        });
+        if (this._doVerification && !this._started) {
+            this._started = true;
+            Promise.resolve(this._doVerification())
+                .then(this.done.bind(this), this.cancel.bind(this));
+        }
+        return this._promise;
+    }
+
+    async _verifyKeys(userId, keys, verifier) {
+        for (const [keyId, keyInfo] of Object.entries(keys)) {
+            const deviceId = keyId.split(':', 2)[1];
+            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            if (!device) {
+                throw new Error(`Could not find device ${deviceId}`);
+            } else {
+                await verifier(keyId, device, keyInfo);
+            }
+        }
+        for (const keyId of Object.keys(keys)) {
+            const deviceId = keyId.split(':', 2)[1];
+            await this._baseApis.setDeviceVerified(userId, deviceId);
+        }
+    }
+}
@@ -0,0 +1,87 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Error messages.
+ *
+ * @module crypto/verification/Error
+ */
+
+import {MatrixEvent} from "../../models/event";
+
+export function newVerificationError(code, reason, extradata) {
+    extradata = extradata || {};
+    extradata.code = code;
+    extradata.reason = reason;
+    return new MatrixEvent({
+        type: "m.key.verification.cancel",
+        content: extradata,
+    });
+}
+
+export function errorFactory(code, reason) {
+    return function(extradata) {
+        return newVerificationError(code, reason, extradata);
+    };
+}
+
+/**
+ * The verification was cancelled by the user.
+ */
+export const newUserCancelledError = errorFactory("m.user", "Cancelled by user");
+
+/**
+ * The verification timed out.
+ */
+export const newTimeoutError = errorFactory("m.timeout", "Timed out");
+
+/**
+ * The transaction is unknown.
+ */
+export const newUnknownTransactionError = errorFactory(
+    "m.unknown_transaction", "Unknown transaction",
+);
+
+/**
+ * An unknown method was selected.
+ */
+export const newUnknownMethodError = errorFactory("m.unknown_method", "Unknown method");
+
+/**
+ * An unexpected message was sent.
+ */
+export const newUnexpectedMessageError = errorFactory(
+    "m.unexpected_message", "Unexpected message",
+);
+
+/**
+ * The key does not match.
+ */
+export const newKeyMismatchError = errorFactory(
+    "m.key_mismatch", "Key mismatch",
+);
+
+/**
+ * The user does not match.
+ */
+export const newUserMismatchError = errorFactory("m.user_error", "User mismatch");
+
+/**
+ * An invalid message was sent.
+ */
+export const newInvalidMessageError = errorFactory(
+    "m.invalid_message", "Invalid message",
+);
@@ -0,0 +1,123 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * QR code key verification.
+ * @module crypto/verification/QRCode
+ */
+
+import Base from "./Base";
+import {
+    errorFactory,
+    newUserCancelledError,
+    newKeyMismatchError,
+    newUserMismatchError,
+} from './Error';
+
+const MATRIXTO_REGEXP = /^(?:https?:\/\/)?(?:www\.)?matrix\.to\/#\/([#@!+][^?]+)\?(.+)$/;
+const KEY_REGEXP = /^key_([^:]+:.+)$/;
+
+const newQRCodeError = errorFactory("m.qr_code.invalid", "Invalid QR code");
+
+/**
+ * @class crypto/verification/QRCode/ShowQRCode
+ * @extends {module:crypto/verification/Base}
+ */
+export class ShowQRCode extends Base {
+    _doVerification() {
+        if (!this._done) {
+            const url = "https://matrix.to/#/" + this._baseApis.getUserId()
+                  + "?device=" + encodeURIComponent(this._baseApis.deviceId)
+                  + "&action=verify&key_ed25519%3A"
+                  + encodeURIComponent(this._baseApis.deviceId) + "="
+                  + encodeURIComponent(this._baseApis.getDeviceEd25519Key());
+            this.emit("show_qr_code", {
+                url: url,
+            });
+        }
+    }
+}
+
+ShowQRCode.NAME = "m.qr_code.show.v1";
+
+/**
+ * @class crypto/verification/QRCode/ScanQRCode
+ * @extends {module:crypto/verification/Base}
+ */
+export class ScanQRCode extends Base {
+    static factory(...args) {
+        return new ScanQRCode(...args);
+    }
+
+    async _doVerification() {
+        const code = await new Promise((resolve, reject) => {
+            this.emit("scan", {
+                done: resolve,
+                cancel: () => reject(newUserCancelledError()),
+            });
+        });
+
+        const match = code.match(MATRIXTO_REGEXP);
+        let deviceId;
+        const keys = {};
+        if (!match) {
+            throw newQRCodeError();
+        }
+        const userId = match[1];
+        const params = match[2].split("&").map(
+            (x) => x.split("=", 2).map(decodeURIComponent),
+        );
+        let action;
+        for (const [name, value] of params) {
+            if (name === "device") {
+                deviceId = value;
+            } else if (name === "action") {
+                action = value;
+            } else {
+                const keyMatch = name.match(KEY_REGEXP);
+                if (keyMatch) {
+                    keys[keyMatch[1]] = value;
+                }
+            }
+        }
+        if (!deviceId || action !== "verify" || Object.keys(keys).length === 0) {
+            throw newQRCodeError();
+        }
+
+        if (!this.userId) {
+            await new Promise((resolve, reject) => {
+                this.emit("confirm_user_id", {
+                    userId: userId,
+                    confirm: resolve,
+                    cancel: () => reject(newUserMismatchError()),
+                });
+            });
+        } else if (this.userId !== userId) {
+            throw newUserMismatchError({
+                expected: this.userId,
+                actual: userId,
+            });
+        }
+
+        await this._verifyKeys(userId, keys, (keyId, device, key) => {
+            if (device.keys[keyId] !== key) {
+                throw newKeyMismatchError();
+            }
+        });
+    }
+}
+
+ScanQRCode.NAME = "m.qr_code.scan.v1";
@@ -0,0 +1,368 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Short Authentication String (SAS) verification.
+ * @module crypto/verification/SAS
+ */
+
+import Base from "./Base";
+import anotherjson from 'another-json';
+import {
+    errorFactory,
+    newUserCancelledError,
+    newUnknownMethodError,
+    newKeyMismatchError,
+    newInvalidMessageError,
+} from './Error';
+
+const EVENTS = [
+    "m.key.verification.accept",
+    "m.key.verification.key",
+    "m.key.verification.mac",
+];
+
+let olmutil;
+
+const newMismatchedSASError = errorFactory(
+    "m.mismatched_sas", "Mismatched short authentication string",
+);
+
+const newMismatchedCommitmentError = errorFactory(
+    "m.mismatched_commitment", "Mismatched commitment",
+);
+
+function generateDecimalSas(sasBytes) {
+    /**
+     *      +--------+--------+--------+--------+--------+
+     *      | Byte 0 | Byte 1 | Byte 2 | Byte 3 | Byte 4 |
+     *      +--------+--------+--------+--------+--------+
+     * bits: 87654321 87654321 87654321 87654321 87654321
+     *       \____________/\_____________/\____________/
+     *         1st number    2nd number     3rd number
+     */
+    return [
+        (sasBytes[0] << 5 | sasBytes[1] >> 3) + 1000,
+        ((sasBytes[1] & 0x7) << 10 | sasBytes[2] << 2 | sasBytes[3] >> 6) + 1000,
+        ((sasBytes[3] & 0x3f) << 7 | sasBytes[4] >> 1) + 1000,
+    ];
+}
+
+const emojiMapping = [
+    ["🐶", "dog"],        //  0
+    ["🐱", "cat"],        //  1
+    ["🦁", "lion"],       //  2
+    ["🐎", "horse"],      //  3
+    ["🦄", "unicorn"],    //  4
+    ["🐷", "pig"],        //  5
+    ["🐘", "elephant"],   //  6
+    ["🐰", "rabbit"],     //  7
+    ["🐼", "panda"],      //  8
+    ["🐓", "rooster"],    //  9
+    ["🐧", "penguin"],    // 10
+    ["🐢", "turtle"],     // 11
+    ["🐟", "fish"],       // 12
+    ["🐙", "octopus"],    // 13
+    ["🦋", "butterfly"],  // 14
+    ["🌷", "flower"],     // 15
+    ["🌳", "tree"],       // 16
+    ["🌵", "cactus"],     // 17
+    ["🍄", "mushroom"],   // 18
+    ["🌏", "globe"],      // 19
+    ["🌙", "moon"],       // 20
+    ["☁️", "cloud"],       // 21
+    ["🔥", "fire"],       // 22
+    ["🍌", "banana"],     // 23
+    ["🍎", "apple"],      // 24
+    ["🍓", "strawberry"], // 25
+    ["🌽", "corn"],       // 26
+    ["🍕", "pizza"],      // 27
+    ["🎂", "cake"],       // 28
+    ["❤️", "heart"],      // 29
+    ["🙂", "smiley"],      // 30
+    ["🤖", "robot"],      // 31
+    ["🎩", "hat"],        // 32
+    ["👓", "glasses"],    // 33
+    ["🔧", "spanner"],     // 34
+    ["🎅", "santa"],      // 35
+    ["👍", "thumbs up"],  // 36
+    ["☂️", "umbrella"],    // 37
+    ["⌛", "hourglass"],   // 38
+    ["⏰", "clock"],      // 39
+    ["🎁", "gift"],       // 40
+    ["💡", "light bulb"], // 41
+    ["📕", "book"],       // 42
+    ["✏️", "pencil"],     // 43
+    ["📎", "paperclip"],  // 44
+    ["✂️", "scissors"],    // 45
+    ["🔒", "padlock"],    // 46
+    ["🔑", "key"],        // 47
+    ["🔨", "hammer"],     // 48
+    ["☎️", "telephone"],  // 49
+    ["🏁", "flag"],       // 50
+    ["🚂", "train"],      // 51
+    ["🚲", "bicycle"],    // 52
+    ["✈️", "aeroplane"],   // 53
+    ["🚀", "rocket"],     // 54
+    ["🏆", "trophy"],     // 55
+    ["⚽", "ball"],       // 56
+    ["🎸", "guitar"],     // 57
+    ["🎺", "trumpet"],    // 58
+    ["🔔", "bell"],       // 59
+    ["⚓️", "anchor"],     // 60
+    ["🎧", "headphones"], // 61
+    ["📁", "folder"],     // 62
+    ["📌", "pin"],        // 63
+];
+
+function generateEmojiSas(sasBytes) {
+    const emojis = [
+        // just like base64 encoding
+        sasBytes[0] >> 2,
+        (sasBytes[0] & 0x3) << 4 | sasBytes[1] >> 4,
+        (sasBytes[1] & 0xf) << 2 | sasBytes[2] >> 6,
+        sasBytes[2] & 0x3f,
+        sasBytes[3] >> 2,
+        (sasBytes[3] & 0x3) << 4 | sasBytes[4] >> 4,
+        (sasBytes[4] & 0xf) << 2 | sasBytes[5] >> 6,
+    ];
+
+    return emojis.map((num) => emojiMapping[num]);
+}
+
+function generateSas(sasBytes, methods) {
+    const sas = {};
+    if (methods.includes("decimal")) {
+        sas["decimal"] = generateDecimalSas(sasBytes);
+    }
+    if (methods.includes("emoji")) {
+        sas["emoji"] = generateEmojiSas(sasBytes);
+    }
+    return sas;
+}
+
+/**
+ * @alias module:crypto/verification/SAS
+ * @extends {module:crypto/verification/Base}
+ */
+export default class SAS extends Base {
+    get events() {
+        return EVENTS;
+    }
+
+    async _doVerification() {
+        await global.Olm.init();
+        olmutil = olmutil || new global.Olm.Utility();
+
+        // make sure user's keys are downloaded
+        await this._baseApis.downloadKeys([this.userId]);
+
+        if (this.startEvent) {
+            return await this._doRespondVerification();
+        } else {
+            return await this._doSendVerification();
+        }
+    }
+
+    async _doSendVerification() {
+        const initialMessage = {
+            method: SAS.NAME,
+            from_device: this._baseApis.deviceId,
+            key_agreement_protocols: ["curve25519"],
+            hashes: ["sha256"],
+            message_authentication_codes: ["hmac-sha256"],
+            // FIXME: allow app to specify what SAS methods can be used
+            short_authentication_string: ["decimal", "emoji"],
+            transaction_id: this.transactionId,
+        };
+        this._sendToDevice("m.key.verification.start", initialMessage);
+
+
+        let e = await this._waitForEvent("m.key.verification.accept");
+        let content = e.getContent();
+        if (!(content.key_agreement_protocol === "curve25519"
+              && content.hash === "sha256"
+              && content.message_authentication_code === "hmac-sha256"
+              && content.short_authentication_string instanceof Array
+              && (content.short_authentication_string.includes("decimal")
+                  || content.short_authentication_string.includes("emoji")))) {
+            throw newUnknownMethodError();
+        }
+        if (typeof content.commitment !== "string") {
+            throw newInvalidMessageError();
+        }
+        const hashCommitment = content.commitment;
+        const sasMethods = content.short_authentication_string;
+        const olmSAS = new global.Olm.SAS();
+        try {
+            this._sendToDevice("m.key.verification.key", {
+                key: olmSAS.get_pubkey(),
+            });
+
+
+            e = await this._waitForEvent("m.key.verification.key");
+            // FIXME: make sure event is properly formed
+            content = e.getContent();
+            const commitmentStr = content.key + anotherjson.stringify(initialMessage);
+            if (olmutil.sha256(commitmentStr) !== hashCommitment) {
+                throw newMismatchedCommitmentError();
+            }
+            olmSAS.set_their_key(content.key);
+
+            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
+                  + this._baseApis.getUserId() + this._baseApis.deviceId
+                  + this.userId + this.deviceId
+                  + this.transactionId;
+            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const verifySAS = new Promise((resolve, reject) => {
+                this.emit("show_sas", {
+                    sas: generateSas(sasBytes, sasMethods),
+                    confirm: () => {
+                        this._sendMAC(olmSAS);
+                        resolve();
+                    },
+                    cancel: () => reject(newUserCancelledError()),
+                    mismatch: () => reject(newMismatchedSASError()),
+                });
+            });
+
+
+            [e] = await Promise.all([
+                this._waitForEvent("m.key.verification.mac"),
+                verifySAS,
+            ]);
+            content = e.getContent();
+            await this._checkMAC(olmSAS, content);
+        } finally {
+            olmSAS.free();
+        }
+    }
+
+    async _doRespondVerification() {
+        let content = this.startEvent.getContent();
+        if (!(content.key_agreement_protocols instanceof Array
+              && content.key_agreement_protocols.includes("curve25519")
+              && content.hashes instanceof Array
+              && content.hashes.includes("sha256")
+              && content.message_authentication_codes instanceof Array
+              && content.message_authentication_codes.includes("hmac-sha256")
+              && content.short_authentication_string instanceof Array
+              && (content.short_authentication_string.includes("decimal")
+                  || content.short_authentication_string.includes("emoji")))) {
+            throw newUnknownMethodError();
+        }
+
+        const olmSAS = new global.Olm.SAS();
+        const sasMethods = [];
+        // FIXME: allow app to specify what SAS methods can be used
+        if (content.short_authentication_string.includes("decimal")) {
+            sasMethods.push("decimal");
+        }
+        if (content.short_authentication_string.includes("emoji")) {
+            sasMethods.push("emoji");
+        }
+        try {
+            const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
+            this._sendToDevice("m.key.verification.accept", {
+                key_agreement_protocol: "curve25519",
+                hash: "sha256",
+                message_authentication_code: "hmac-sha256",
+                short_authentication_string: sasMethods,
+                commitment: olmutil.sha256(commitmentStr),
+            });
+
+
+            let e = await this._waitForEvent("m.key.verification.key");
+            // FIXME: make sure event is properly formed
+            content = e.getContent();
+            olmSAS.set_their_key(content.key);
+            this._sendToDevice("m.key.verification.key", {
+                key: olmSAS.get_pubkey(),
+            });
+
+            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
+                  + this.userId + this.deviceId
+                  + this._baseApis.getUserId() + this._baseApis.deviceId
+                  + this.transactionId;
+            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const verifySAS = new Promise((resolve, reject) => {
+                this.emit("show_sas", {
+                    sas: generateSas(sasBytes, sasMethods),
+                    confirm: () => {
+                        this._sendMAC(olmSAS);
+                        resolve();
+                    },
+                    cancel: () => reject(newUserCancelledError()),
+                    mismatch: () => reject(newMismatchedSASError()),
+                });
+            });
+
+
+            [e] = await Promise.all([
+                this._waitForEvent("m.key.verification.mac"),
+                verifySAS,
+            ]);
+            content = e.getContent();
+            await this._checkMAC(olmSAS, content);
+        } finally {
+            olmSAS.free();
+        }
+    }
+
+    _sendMAC(olmSAS) {
+        const keyId = `ed25519:${this._baseApis.deviceId}`;
+        const mac = {};
+        const baseInfo = "MATRIX_KEY_VERIFICATION_MAC"
+              + this._baseApis.getUserId() + this._baseApis.deviceId
+              + this.userId + this.deviceId
+              + this.transactionId;
+
+        mac[keyId] = olmSAS.calculate_mac(
+            this._baseApis.getDeviceEd25519Key(),
+            baseInfo + keyId,
+        );
+        const keys = olmSAS.calculate_mac(
+            keyId,
+            baseInfo + "KEY_IDS",
+        );
+        this._sendToDevice("m.key.verification.mac", { mac, keys });
+    }
+
+    async _checkMAC(olmSAS, content) {
+        const baseInfo = "MATRIX_KEY_VERIFICATION_MAC"
+              + this.userId + this.deviceId
+              + this._baseApis.getUserId() + this._baseApis.deviceId
+              + this.transactionId;
+
+        if (content.keys !== olmSAS.calculate_mac(
+            Object.keys(content.mac).sort().join(","),
+            baseInfo + "KEY_IDS",
+        )) {
+            throw newKeyMismatchError();
+        }
+
+        await this._verifyKeys(this.userId, content.mac, (keyId, device, keyInfo) => {
+            if (keyInfo !== olmSAS.calculate_mac(
+                device.keys[keyId],
+                baseInfo + keyId,
+            )) {
+                throw newKeyMismatchError();
+            }
+        });
+    }
+}
+
+SAS.NAME = "m.sas.v1";
@@ -0,0 +1,46 @@
+// can't just do InvalidStoreError extends Error
+// because of http://babeljs.io/docs/usage/caveats/#classes
+export function InvalidStoreError(reason, value) {
+    const message = `Store is invalid because ${reason}, ` +
+        `please stop the client, delete all data and start the client again`;
+    const instance = Reflect.construct(Error, [message]);
+    Reflect.setPrototypeOf(instance, Reflect.getPrototypeOf(this));
+    instance.reason = reason;
+    instance.value = value;
+    return instance;
+}
+
+InvalidStoreError.TOGGLED_LAZY_LOADING = "TOGGLED_LAZY_LOADING";
+
+InvalidStoreError.prototype = Object.create(Error.prototype, {
+  constructor: {
+    value: Error,
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  },
+});
+Reflect.setPrototypeOf(InvalidStoreError, Error);
+
+
+export function InvalidCryptoStoreError(reason) {
+    const message = `Crypto store is invalid because ${reason}, ` +
+        `please stop the client, delete all data and start the client again`;
+    const instance = Reflect.construct(Error, [message]);
+    Reflect.setPrototypeOf(instance, Reflect.getPrototypeOf(this));
+    instance.reason = reason;
+    instance.name = 'InvalidCryptoStoreError';
+    return instance;
+}
+
+InvalidCryptoStoreError.TOO_NEW = "TOO_NEW";
+
+InvalidCryptoStoreError.prototype = Object.create(Error.prototype, {
+  constructor: {
+    value: Error,
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  },
+});
+Reflect.setPrototypeOf(InvalidCryptoStoreError, Error);
@@ -97,11 +97,12 @@ FilterComponent.prototype._checkFields =
    };

    const self = this;
-    Object.keys(literal_keys).forEach(function(name) {
+    for (let n=0; n < Object.keys(literal_keys).length; n++) {
+        const name = Object.keys(literal_keys)[n];
        const match_func = literal_keys[name];
        const not_name = "not_" + name;
        const disallowed_values = self[not_name];
-        if (disallowed_values.map(match_func)) {
+        if (disallowed_values.filter(match_func).length > 0) {
            return false;
        }

@@ -111,7 +112,7 @@ FilterComponent.prototype._checkFields =
                return false;
            }
        }
-    });
+    }

    const contains_url_filter = this.filter_json.contains_url;
    if (contains_url_filter !== undefined) {
@@ -51,6 +51,17 @@ function Filter(userId, filterId) {
    this.definition = {};
 }

+Filter.LAZY_LOADING_MESSAGES_FILTER = {
+    lazy_load_members: true,
+};
+
+Filter.LAZY_LOADING_SYNC_FILTER = {
+    room: {
+        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
+    },
+};
+
+
 /**
 * Get the ID of this filter on your homeserver (if known)
 * @return {?Number} The filter ID
@@ -18,7 +18,9 @@ limitations under the License.
 * This is an internal module. See {@link MatrixHttpApi} for the public class.
 * @module http-api
 */
-const q = require("q");
+import Promise from 'bluebird';
+const parseContentType = require('content-type').parse;
+
 const utils = require("./utils");

 // we use our own implementation of setTimeout, so that if we get suspended in
@@ -64,7 +66,7 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
 * @param {string} opts.prefix Required. The matrix client prefix to use, e.g.
 * '/_matrix/client/r0'. See PREFIX_R0 and PREFIX_UNSTABLE for constants.
 *
- * @param {bool=} opts.onlyData True to return only the 'data' component of the
+ * @param {boolean} opts.onlyData True to return only the 'data' component of the
 * response (e.g. the parsed HTTP body). If false, requests will return an
 * object with the properties <tt>code</tt>, <tt>headers</tt> and <tt>data</tt>.
 *
@@ -74,12 +76,15 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
 * requests.
 * @param {Number=} opts.localTimeoutMs The default maximum amount of time to wait
 * before timing out the request. If not specified, there is no timeout.
+ * @param {boolean} [opts.useAuthorizationHeader = false] Set to true to use
+ * Authorization header instead of query param to send the access token to the server.
 */
 module.exports.MatrixHttpApi = function MatrixHttpApi(event_emitter, opts) {
    utils.checkObjectHasKeys(opts, ["baseUrl", "request", "prefix"]);
    opts.onlyData = opts.onlyData || false;
    this.event_emitter = event_emitter;
    this.opts = opts;
+    this.useAuthorizationHeader = Boolean(opts.useAuthorizationHeader);
    this.uploads = [];
 };

@@ -113,6 +118,10 @@ module.exports.MatrixHttpApi.prototype = {
     * @param {string=} opts.name   Name to give the file on the server. Defaults
     *   to <tt>file.name</tt>.
     *
+     * @param {boolean=} opts.includeFilename if false will not send the filename,
+     *   e.g for encrypted file uploads where filename leaks are undesirable.
+     *   Defaults to true.
+     *
     * @param {string=} opts.type   Content-type for the upload. Defaults to
     *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
     *
@@ -129,6 +138,10 @@ module.exports.MatrixHttpApi.prototype = {
     *    invoke on success/failure. See the promise return values for more
     *    information.
     *
+     * @param {Function=} opts.progressHandler Optional. Called when a chunk of
+     *    data has been uploaded, with an object containing the fields `loaded`
+     *    (number of bytes transferred) and `total` (total size, if known).
+     *
     * @return {module:client.Promise} Resolves to response object, as
     *    determined by this.opts.onlyData, opts.rawResponse, and
     *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
@@ -143,6 +156,9 @@ module.exports.MatrixHttpApi.prototype = {
            opts = {};
        }

+        // default opts.includeFilename to true (ignoring falsey values)
+        const includeFilename = opts.includeFilename !== false;
+
        // if the file doesn't have a mime type, use a default since
        // the HS errors if we don't supply one.
        const contentType = opts.type || file.type || 'application/octet-stream';
@@ -214,7 +230,7 @@ module.exports.MatrixHttpApi.prototype = {
        }

        if (global.XMLHttpRequest) {
-            const defer = q.defer();
+            const defer = Promise.defer();
            const xhr = new global.XMLHttpRequest();
            upload.xhr = xhr;
            const cb = requestCallback(defer, opts.callback, this.opts.onlyData);
@@ -255,13 +271,34 @@ module.exports.MatrixHttpApi.prototype = {
                upload.loaded = ev.loaded;
                upload.total = ev.total;
                xhr.timeout_timer = callbacks.setTimeout(timeout_fn, 30000);
-                defer.notify(ev);
+                if (opts.progressHandler) {
+                    opts.progressHandler({
+                        loaded: ev.loaded,
+                        total: ev.total,
+                    });
+                }
            });
            let url = this.opts.baseUrl + "/_matrix/media/v1/upload";
-            url += "?access_token=" + encodeURIComponent(this.opts.accessToken);
-            url += "&filename=" + encodeURIComponent(fileName);
+
+            const queryArgs = [];
+
+            if (includeFilename && fileName) {
+                queryArgs.push("filename=" + encodeURIComponent(fileName));
+            }
+
+            if (!this.useAuthorizationHeader) {
+                queryArgs.push("access_token="
+                    + encodeURIComponent(this.opts.accessToken));
+            }
+
+            if (queryArgs.length > 0) {
+                url += "?" + queryArgs.join("&");
+            }

            xhr.open("POST", url);
+            if (this.useAuthorizationHeader) {
+                xhr.setRequestHeader("Authorization", "Bearer " + this.opts.accessToken);
+            }
            xhr.setRequestHeader("Content-Type", contentType);
            xhr.send(body);
            promise = defer.promise;
@@ -269,9 +306,11 @@ module.exports.MatrixHttpApi.prototype = {
            // dirty hack (as per _request) to allow the upload to be cancelled.
            promise.abort = xhr.abort.bind(xhr);
        } else {
-            const queryParams = {
-                filename: fileName,
-            };
+            const queryParams = {};
+
+            if (includeFilename && fileName) {
+                queryParams.filename = fileName;
+            }

            promise = this.authedRequest(
                opts.callback, "POST", "/upload", queryParams, body, {
@@ -338,7 +377,7 @@ module.exports.MatrixHttpApi.prototype = {
            opts.form = params;
        }

-        const defer = q.defer();
+        const defer = Promise.defer();
        this.opts.request(
            opts,
            requestCallback(defer, callback, this.opts.onlyData),
@@ -364,7 +403,8 @@ module.exports.MatrixHttpApi.prototype = {
     *
     * @param {Object} data The HTTP JSON body.
     *
-     * @param {Object=} opts additional options
+     * @param {Object|Number=} opts additional options. If a number is specified,
+     * this is treated as `opts.localTimeoutMs`.
     *
     * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
     * timing out the request. If not specified, there is no timeout.
@@ -385,24 +425,51 @@ module.exports.MatrixHttpApi.prototype = {
        if (!queryParams) {
            queryParams = {};
        }
-        if (!queryParams.access_token) {
-            queryParams.access_token = this.opts.accessToken;
+        if (this.useAuthorizationHeader) {
+            if (isFinite(opts)) {
+                // opts used to be localTimeoutMs
+                opts = {
+                    localTimeoutMs: opts,
+                };
+            }
+            if (!opts) {
+                opts = {};
+            }
+            if (!opts.headers) {
+                opts.headers = {};
+            }
+            if (!opts.headers.Authorization) {
+                opts.headers.Authorization = "Bearer " + this.opts.accessToken;
+            }
+            if (queryParams.access_token) {
+                delete queryParams.access_token;
+            }
+        } else {
+            if (!queryParams.access_token) {
+                queryParams.access_token = this.opts.accessToken;
+            }
        }

-        const request_promise = this.request(
+        const requestPromise = this.request(
            callback, method, path, queryParams, data, opts,
        );

        const self = this;
-        request_promise.catch(function(err) {
+        requestPromise.catch(function(err) {
            if (err.errcode == 'M_UNKNOWN_TOKEN') {
                self.event_emitter.emit("Session.logged_out");
+            } else if (err.errcode == 'M_CONSENT_NOT_GIVEN') {
+                self.event_emitter.emit(
+                    "no_consent",
+                    err.message,
+                    err.data.consent_uri,
+                );
            }
        });

        // return the original promise, otherwise tests break due to it having to
        // go around the event loop one more time to process the result of the request
-        return request_promise;
+        return requestPromise;
    },

    /**
@@ -623,9 +690,33 @@ module.exports.MatrixHttpApi.prototype = {
            }
        }

+        const headers = utils.extend({}, opts.headers || {});
        const json = opts.json === undefined ? true : opts.json;
+        let bodyParser = opts.bodyParser;

-        const defer = q.defer();
+        // we handle the json encoding/decoding here, because request and
+        // browser-request make a mess of it. Specifically, they attempt to
+        // json-decode plain-text error responses, which in turn means that the
+        // actual error gets swallowed by a SyntaxError.
+
+        if (json) {
+            if (data) {
+                data = JSON.stringify(data);
+                headers['content-type'] = 'application/json';
+            }
+
+            if (!headers['accept']) {
+                headers['accept'] = 'application/json';
+            }
+
+            if (bodyParser === undefined) {
+                bodyParser = function(rawBody) {
+                    return JSON.parse(rawBody);
+                };
+            }
+        }
+
+        const defer = Promise.defer();

        let timeoutId;
        let timedOut = false;
@@ -661,10 +752,12 @@ module.exports.MatrixHttpApi.prototype = {
                    method: method,
                    withCredentials: false,
                    qs: queryParams,
+                    qsStringifyOptions: opts.qsStringifyOptions,
+                    useQuerystring: true,
                    body: data,
-                    json: json,
+                    json: false,
                    timeout: localTimeoutMs,
-                    headers: opts.headers || {},
+                    headers: headers || {},
                    _matrix_opts: this.opts,
                },
                function(err, response, body) {
@@ -675,13 +768,9 @@ module.exports.MatrixHttpApi.prototype = {
                        }
                    }

-                    // if json is falsy, we won't parse any error response, so need
-                    // to do so before turning it into a MatrixError
-                    const parseErrorJson = !json;
                    const handlerFn = requestCallback(
                        defer, callback, self.opts.onlyData,
-                        parseErrorJson,
-                        opts.bodyParser,
+                        bodyParser,
                    );
                    handlerFn(err, response, body);
                },
@@ -718,15 +807,18 @@ module.exports.MatrixHttpApi.prototype = {
 * that will either resolve or reject the given defer as well as invoke the
 * given userDefinedCallback (if any).
 *
- * If onlyData is true, the defer/callback is invoked with the body of the
- * response, otherwise the result code.
+ * HTTP errors are transformed into javascript errors and the deferred is rejected.
 *
- * If parseErrorJson is true, we will JSON.parse the body if we get a 4xx error.
+ * If bodyParser is given, it is used to transform the body of the successful
+ * responses before passing to the defer/callback.
+ *
+ * If onlyData is true, the defer/callback is invoked with the body of the
+ * response, otherwise the result object (with `code` and `data` fields)
 *
 */
 const requestCallback = function(
    defer, userDefinedCallback, onlyData,
-    parseErrorJson, bodyParser,
+    bodyParser,
 ) {
    userDefinedCallback = userDefinedCallback || function() {};

@@ -734,19 +826,12 @@ const requestCallback = function(
        if (!err) {
            try {
                if (response.statusCode >= 400) {
-                    if (parseErrorJson) {
-                        // we won't have json-decoded the response.
-                        body = JSON.parse(body);
-                    }
-                    err = new module.exports.MatrixError(body);
+                    err = parseErrorResponse(response, body);
                } else if (bodyParser) {
                    body = bodyParser(body);
                }
            } catch (e) {
-                err = e;
-            }
-            if (err) {
-                err.httpStatus = response.statusCode;
+                err = new Error(`Error parsing server response: ${e}`);
            }
        }

@@ -756,6 +841,9 @@ const requestCallback = function(
        } else {
            const res = {
                code: response.statusCode,
+
+                // XXX: why do we bother with this? it doesn't work for
+                // XMLHttpRequest, so clearly we don't use it.
                headers: response.headers,
                data: body,
            };
@@ -765,6 +853,67 @@ const requestCallback = function(
    };
 };

+/**
+ * Attempt to turn an HTTP error response into a Javascript Error.
+ *
+ * If it is a JSON response, we will parse it into a MatrixError. Otherwise
+ * we return a generic Error.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @param {String} body raw body of the response
+ * @returns {Error}
+ */
+function parseErrorResponse(response, body) {
+    const httpStatus = response.statusCode;
+    const contentType = getResponseContentType(response);
+
+    let err;
+    if (contentType) {
+        if (contentType.type === 'application/json') {
+            err = new module.exports.MatrixError(JSON.parse(body));
+        } else if (contentType.type === 'text/plain') {
+            err = new Error(`Server returned ${httpStatus} error: ${body}`);
+        }
+    }
+
+    if (!err) {
+        err = new Error(`Server returned ${httpStatus} error`);
+    }
+    err.httpStatus = httpStatus;
+    return err;
+}
+
+
+/**
+ * extract the Content-Type header from the response object, and
+ * parse it to a `{type, parameters}` object.
+ *
+ * returns null if no content-type header could be found.
+ *
+ * @param {XMLHttpRequest|http.IncomingMessage} response response object
+ * @returns {{type: String, parameters: Object}?} parsed content-type header, or null if not found
+ */
+function getResponseContentType(response) {
+    let contentType;
+    if (response.getResponseHeader) {
+        // XMLHttpRequest provides getResponseHeader
+        contentType = response.getResponseHeader("Content-Type");
+    } else if (response.headers) {
+        // request provides http.IncomingMessage which has a message.headers map
+        contentType = response.headers['content-type'] || null;
+    }
+
+    if (!contentType) {
+        return null;
+    }
+
+    try {
+        return parseContentType(contentType);
+    } catch(e) {
+        throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
+    }
+}
+
 /**
 * Construct a Matrix error. This is a JavaScript Error with additional
 * information specific to the standard Matrix error response.
@@ -17,7 +17,7 @@ limitations under the License.
 "use strict";

 /** @module interactive-auth */
-const q = require("q");
+import Promise from 'bluebird';
 const url = require("url");

 const utils = require("./utils");
@@ -115,11 +115,11 @@ InteractiveAuth.prototype = {
     *     no suitable authentication flow can be found
     */
    attemptAuth: function() {
-        this._completionDeferred = q.defer();
+        this._completionDeferred = Promise.defer();

        // wrap in a promise so that if _startNextAuthStage
        // throws, it rejects the promise in a consistent way
-        return q().then(() => {
+        return Promise.resolve().then(() => {
            // if we have no flows, try a request (we'll have
            // just a session ID in _data if resuming)
            if (!this._data.flows) {
@@ -258,12 +258,12 @@ InteractiveAuth.prototype = {

        // hackery to make sure that synchronous exceptions end up in the catch
        // handler (without the additional event loop entailed by q.fcall or an
-        // extra q().then)
+        // extra Promise.resolve().then)
        let prom;
        try {
            prom = this._requestCallback(auth, background);
        } catch (e) {
-            prom = q.reject(e);
+            prom = Promise.reject(e);
        }

        prom = prom.then(
@@ -293,7 +293,9 @@ InteractiveAuth.prototype = {
            },
        );
        if (!background) {
-            prom = prom.catch(this._completionDeferred.reject);
+            prom = prom.catch((e) => {
+                this._completionDeferred.reject(e);
+            });
        } else {
            // We ignore all failures here (even non-UI auth related ones)
            // since we don't want to suddenly fail if the internet connection
@@ -0,0 +1,36 @@
+/*
+Copyright 2018 André Jaenisch
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module logger
+ */
+const log = require("loglevel");
+
+// This is to demonstrate, that you can use any namespace you want.
+// Namespaces allow you to turn on/off the logging for specific parts of the
+// application.
+// An idea would be to control this via an environment variable (on Node.js).
+// See https://www.npmjs.com/package/debug to see how this could be implemented
+// Part of #332 is introducing a logging library in the first place.
+const DEFAULT_NAME_SPACE = "matrix";
+const logger = log.getLogger(DEFAULT_NAME_SPACE);
+logger.setLevel(log.levels.DEBUG);
+
+/**
+ * Drop-in replacement for <code>console</code> using {@link https://www.npmjs.com/package/loglevel|loglevel}.
+ * Can be tailored down to specific use cases if needed.
+*/
+module.exports = logger;
@@ -16,6 +16,8 @@ limitations under the License.
 */
 "use strict";

+/** The {@link module:ContentHelpers} object */
+module.exports.ContentHelpers = require("./content-helpers");
 /** The {@link module:models/event.MatrixEvent|MatrixEvent} class. */
 module.exports.MatrixEvent = require("./models/event").MatrixEvent;
 /** The {@link module:models/event.EventStatus|EventStatus} enum. */
@@ -32,10 +34,14 @@ module.exports.SyncAccumulator = require("./sync-accumulator");
 module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi;
 /** The {@link module:http-api.MatrixError|MatrixError} class. */
 module.exports.MatrixError = require("./http-api").MatrixError;
+/** The {@link module:errors.InvalidStoreError|InvalidStoreError} class. */
+module.exports.InvalidStoreError = require("./errors").InvalidStoreError;
 /** The {@link module:client.MatrixClient|MatrixClient} class. */
 module.exports.MatrixClient = require("./client").MatrixClient;
 /** The {@link module:models/room|Room} class. */
 module.exports.Room = require("./models/room");
+/** The {@link module:models/group|Group} class. */
+module.exports.Group = require("./models/group");
 /** The {@link module:models/event-timeline~EventTimeline} class. */
 module.exports.EventTimeline = require("./models/event-timeline");
 /** The {@link module:models/event-timeline-set~EventTimelineSet} class. */
@@ -61,6 +67,8 @@ module.exports.Filter = require("./filter");
 module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
 /** The {@link module:interactive-auth} class. */
 module.exports.InteractiveAuth = require("./interactive-auth");
+/** The {@link module:auto-discovery|AutoDiscovery} class. */
+module.exports.AutoDiscovery = require("./autodiscovery").AutoDiscovery;


 module.exports.MemoryCryptoStore =
@@ -80,14 +88,21 @@ module.exports.createNewMatrixCall = require("./webrtc/call").createNewMatrixCal


 /**
- * Set an audio input device to use for MatrixCalls
+ * Set a preferred audio output device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallAudioOutput = require('./webrtc/call').setAudioOutput;
+/**
+ * Set a preferred audio input device to use for MatrixCalls
 * @function
 * @param {string=} deviceId the identifier for the device
 * undefined treated as unset
 */
 module.exports.setMatrixCallAudioInput = require('./webrtc/call').setAudioInput;
 /**
- * Set a video input device to use for MatrixCalls
+ * Set a preferred video input device to use for MatrixCalls
 * @function
 * @param {string=} deviceId the identifier for the device
 * undefined treated as unset
@@ -149,44 +149,51 @@ EventTimelineSet.prototype.replaceEventId = function(oldEventId, newEventId) {
 * <p>This is used when /sync returns a 'limited' timeline.
 *
 * @param {string=} backPaginationToken   token for back-paginating the new timeline
- * @param {?bool} flush  Whether to flush the non-live timelines too.
+ * @param {string=} forwardPaginationToken token for forward-paginating the old live timeline,
+ * if absent or null, all timelines are reset.
 *
 * @fires module:client~MatrixClient#event:"Room.timelineReset"
 */
-EventTimelineSet.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
-    // if timeline support is disabled, forget about the old timelines
-    const resetAllTimelines = !this._timelineSupport || flush;
+EventTimelineSet.prototype.resetLiveTimeline = function(
+    backPaginationToken, forwardPaginationToken,
+) {
+    // Each EventTimeline has RoomState objects tracking the state at the start
+    // and end of that timeline. The copies at the end of the live timeline are
+    // special because they will have listeners attached to monitor changes to
+    // the current room state, so we move this RoomState from the end of the
+    // current live timeline to the end of the new one and, if necessary,
+    // replace it with a newly created one. We also make a copy for the start
+    // of the new timeline.
+
+    // if timeline support is disabled, forget about the old timelines
+    const resetAllTimelines = !this._timelineSupport || !forwardPaginationToken;
+
+    const oldTimeline = this._liveTimeline;
+    const newTimeline = resetAllTimelines ?
+        oldTimeline.forkLive(EventTimeline.FORWARDS) :
+        oldTimeline.fork(EventTimeline.FORWARDS);

-    let newTimeline;
    if (resetAllTimelines) {
-        newTimeline = new EventTimeline(this);
        this._timelines = [newTimeline];
        this._eventIdToTimeline = {};
    } else {
-        newTimeline = this.addTimeline();
+        this._timelines.push(newTimeline);
    }

-    // initialise the state in the new timeline from our last known state
-    const evMap = this._liveTimeline.getState(EventTimeline.FORWARDS).events;
-    const events = [];
-    for (const evtype in evMap) {
-        if (!evMap.hasOwnProperty(evtype)) {
-            continue;
-        }
-        for (const stateKey in evMap[evtype]) {
-            if (!evMap[evtype].hasOwnProperty(stateKey)) {
-                continue;
-            }
-            events.push(evMap[evtype][stateKey]);
-        }
+    if (forwardPaginationToken) {
+        // Now set the forward pagination token on the old live timeline
+        // so it can be forward-paginated.
+        oldTimeline.setPaginationToken(
+            forwardPaginationToken, EventTimeline.FORWARDS,
+        );
    }
-    newTimeline.initialiseState(events);

    // make sure we set the pagination token before firing timelineReset,
    // otherwise clients which start back-paginating will fail, and then get
    // stuck without realising that they *can* back-paginate.
    newTimeline.setPaginationToken(backPaginationToken, EventTimeline.BACKWARDS);

+    // Now we can swap the live timeline to the new one.
    this._liveTimeline = newTimeline;
    this.emit("Room.timelineReset", this.room, this, resetAllTimelines);
 };
@@ -20,8 +20,6 @@ limitations under the License.
 */

 const RoomState = require("./room-state");
-const utils = require("../utils");
-const MatrixEvent = require("./event").MatrixEvent;

 /**
 * Construct a new EventTimeline
@@ -88,22 +86,70 @@ EventTimeline.prototype.initialiseState = function(stateEvents) {
        throw new Error("Cannot initialise state after events are added");
    }

-    // we deep-copy the events here, in case they get changed later - we don't
-    // want changes to the start state leaking through to the end state.
-    const oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEvents.map(function(mxEvent) {
-                return mxEvent.event;
-            }),
-        ),
-    function(ev) {
-        return new MatrixEvent(ev);
-    });
+    // We previously deep copied events here and used different copies in
+    // the oldState and state events: this decision seems to date back
+    // quite a way and was apparently made to fix a bug where modifications
+    // made to the start state leaked through to the end state.
+    // This really shouldn't be possible though: the events themselves should
+    // not change. Duplicating the events uses a lot of extra memory,
+    // so we now no longer do it. To assert that they really do never change,
+    // freeze them! Note that we can't do this for events in general:
+    // although it looks like the only things preventing us are the
+    // 'status' flag, forwardLooking (which is only set once when adding to the
+    // timeline) and possibly the sender (which seems like it should never be
+    // reset but in practice causes a lot of the tests to break).
+    for (const e of stateEvents) {
+        Object.freeze(e);
+    }

-    this._startState.setStateEvents(oldStateEvents);
+    this._startState.setStateEvents(stateEvents);
    this._endState.setStateEvents(stateEvents);
 };

+/**
+ * Forks the (live) timeline, taking ownership of the existing directional state of this timeline.
+ * All attached listeners will keep receiving state updates from the new live timeline state.
+ * The end state of this timeline gets replaced with an independent copy of the current RoomState,
+ * and will need a new pagination token if it ever needs to paginate forwards.
+
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.forkLive = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    // Now clobber the end state of the new live timeline with that from the
+    // previous live timeline. It will be identical except that we'll keep
+    // using the same RoomMember objects for the 'live' set of members with any
+    // listeners still attached
+    timeline._endState = forkState;
+    // Firstly, we just stole the current timeline's end state, so it needs a new one.
+    // Make an immutable copy of the state so back pagination will get the correct sentinels.
+    this._endState = forkState.clone();
+    return timeline;
+};
+
+/**
+ * Creates an independent timeline, inheriting the directional state from this timeline.
+ *
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.fork = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    timeline._endState = forkState.clone();
+    return timeline;
+};
+
 /**
 * Get the ID of the room for this timeline
 * @return {string} room ID
@@ -21,9 +21,9 @@ limitations under the License.
 * @module models/event
 */

-const EventEmitter = require("events").EventEmitter;
-
-const utils = require('../utils.js');
+import Promise from 'bluebird';
+import {EventEmitter} from 'events';
+import utils from '../utils.js';

 /**
 * Enum for event statuses.
@@ -80,7 +80,10 @@ module.exports.MatrixEvent = function MatrixEvent(
    // intern the values of matrix events to force share strings and reduce the
    // amount of needless string duplication. This can save moderate amounts of
    // memory (~10% on a 350MB heap).
-    ["state_key", "type", "sender", "room_id"].forEach((prop) => {
+    // 'membership' at the event level (rather than the content level) is a legacy
+    // field that Riot never otherwise looks at, but it will still take up a lot
+    // of space if we don't intern it.
+    ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
        if (!event[prop]) {
            return;
        }
@@ -108,12 +111,35 @@ module.exports.MatrixEvent = function MatrixEvent(
    this.error = null;
    this.forwardLooking = true;
    this._pushActions = null;
-    this._date = this.event.origin_server_ts ?
-        new Date(this.event.origin_server_ts) : null;

    this._clearEvent = {};
-    this._keysProved = {};
-    this._keysClaimed = {};
+
+    /* curve25519 key which we believe belongs to the sender of the event. See
+     * getSenderKey()
+     */
+    this._senderCurve25519Key = null;
+
+    /* ed25519 key which the sender of this event (for olm) or the creator of
+     * the megolm session (for megolm) claims to own. See getClaimedEd25519Key()
+     */
+    this._claimedEd25519Key = null;
+
+    /* curve25519 keys of devices involved in telling us about the
+     * _senderCurve25519Key and _claimedEd25519Key.
+     * See getForwardingCurve25519KeyChain().
+     */
+    this._forwardingCurve25519KeyChain = [];
+
+    /* if we have a process decrypting this event, a Promise which resolves
+     * when it is finished. Normally null.
+     */
+    this._decryptionPromise = null;
+
+    /* flag to indicate if we should retry decrypting this event after the
+     * first attempt (eg, we have received new data which means that a second
+     * attempt may succeed)
+     */
+    this._retryDecryption = false;
 };
 utils.inherits(module.exports.MatrixEvent, EventEmitter);

@@ -179,7 +205,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     * @return {Date} The event date, e.g. <code>new Date(1433502692297)</code>
     */
    getDate: function() {
-        return this._date;
+        return this.event.origin_server_ts ? new Date(this.event.origin_server_ts) : null;
    },

    /**
@@ -261,9 +287,18 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     * <tt>"m.room.encrypted"</tt>
     *
     * @param {object} crypto_content raw 'content' for the encrypted event.
-     * @param {object} keys The local keys claimed and proved by this event.
+     *
+     * @param {string} senderCurve25519Key curve25519 key to record for the
+     *   sender of this event.
+     *   See {@link module:models/event.MatrixEvent#getSenderKey}.
+     *
+     * @param {string} claimedEd25519Key claimed ed25519 key to record for the
+     *   sender if this event.
+     *   See {@link module:models/event.MatrixEvent#getClaimedEd25519Key}
     */
-    makeEncrypted: function(crypto_type, crypto_content, keys) {
+    makeEncrypted: function(
+        crypto_type, crypto_content, senderCurve25519Key, claimedEd25519Key,
+    ) {
        // keep the plain-text data for 'view source'
        this._clearEvent = {
            type: this.event.type,
@@ -271,8 +306,221 @@ utils.extend(module.exports.MatrixEvent.prototype, {
        };
        this.event.type = crypto_type;
        this.event.content = crypto_content;
-        this._keysProved = keys;
-        this._keysClaimed = keys;
+        this._senderCurve25519Key = senderCurve25519Key;
+        this._claimedEd25519Key = claimedEd25519Key;
+    },
+
+    /**
+     * Check if this event is currently being decrypted.
+     *
+     * @return {boolean} True if this event is currently being decrypted, else false.
+     */
+    isBeingDecrypted: function() {
+        return this._decryptionPromise != null;
+    },
+
+    /**
+     * Check if this event is an encrypted event which we failed to decrypt
+     *
+     * (This implies that we might retry decryption at some point in the future)
+     *
+     * @return {boolean} True if this event is an encrypted event which we
+     *     couldn't decrypt.
+     */
+    isDecryptionFailure: function() {
+        return this._clearEvent && this._clearEvent.content &&
+            this._clearEvent.content.msgtype === "m.bad.encrypted";
+    },
+
+    /**
+     * Start the process of trying to decrypt this event.
+     *
+     * (This is used within the SDK: it isn't intended for use by applications)
+     *
+     * @internal
+     *
+     * @param {module:crypto} crypto crypto module
+     *
+     * @returns {Promise} promise which resolves (to undefined) when the decryption
+     * attempt is completed.
+     */
+    attemptDecryption: async function(crypto) {
+        // start with a couple of sanity checks.
+        if (!this.isEncrypted()) {
+            throw new Error("Attempt to decrypt event which isn't encrypted");
+        }
+
+        if (
+            this._clearEvent && this._clearEvent.content &&
+                this._clearEvent.content.msgtype !== "m.bad.encrypted"
+        ) {
+            // we may want to just ignore this? let's start with rejecting it.
+            throw new Error(
+                "Attempt to decrypt event which has already been encrypted",
+            );
+        }
+
+        // if we already have a decryption attempt in progress, then it may
+        // fail because it was using outdated info. We now have reason to
+        // succeed where it failed before, but we don't want to have multiple
+        // attempts going at the same time, so just set a flag that says we have
+        // new info.
+        //
+        if (this._decryptionPromise) {
+            console.log(
+                `Event ${this.getId()} already being decrypted; queueing a retry`,
+            );
+            this._retryDecryption = true;
+            return this._decryptionPromise;
+        }
+
+        this._decryptionPromise = this._decryptionLoop(crypto);
+        return this._decryptionPromise;
+    },
+
+    /**
+     * Cancel any room key request for this event and resend another.
+     *
+     * @param {module:crypto} crypto crypto module
+     * @param {string} userId the user who received this event
+     *
+     * @returns {Promise} a promise that resolves when the request is queued
+     */
+    cancelAndResendKeyRequest: function(crypto, userId) {
+        const wireContent = this.getWireContent();
+        return crypto.requestRoomKey({
+            algorithm: wireContent.algorithm,
+            room_id: this.getRoomId(),
+            session_id: wireContent.session_id,
+            sender_key: wireContent.sender_key,
+        }, this.getKeyRequestRecipients(userId), true);
+    },
+
+    /**
+     * Calculate the recipients for keyshare requests.
+     *
+     * @param {string} userId the user who received this event.
+     *
+     * @returns {Array} array of recipients
+     */
+    getKeyRequestRecipients: function(userId) {
+        // send the request to all of our own devices, and the
+        // original sending device if it wasn't us.
+        const wireContent = this.getWireContent();
+        const recipients = [{
+            userId, deviceId: '*',
+        }];
+        const sender = this.getSender();
+        if (sender !== userId) {
+            recipients.push({
+                userId: sender, deviceId: wireContent.device_id,
+            });
+        }
+        return recipients;
+    },
+
+    _decryptionLoop: async function(crypto) {
+        // make sure that this method never runs completely synchronously.
+        // (doing so would mean that we would clear _decryptionPromise *before*
+        // it is set in attemptDecryption - and hence end up with a stuck
+        // `_decryptionPromise`).
+        await Promise.resolve();
+
+        while (true) {
+            this._retryDecryption = false;
+
+            let res;
+            let err;
+            try {
+                if (!crypto) {
+                    res = this._badEncryptedMessage("Encryption not enabled");
+                } else {
+                    res = await crypto.decryptEvent(this);
+                }
+            } catch (e) {
+                if (e.name !== "DecryptionError") {
+                    // not a decryption error: log the whole exception as an error
+                    // (and don't bother with a retry)
+                    console.error(
+                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                    );
+                    this._decryptionPromise = null;
+                    this._retryDecryption = false;
+                    return;
+                }
+
+                err = e;
+
+                // see if we have a retry queued.
+                //
+                // NB: make sure to keep this check in the same tick of the
+                //   event loop as `_decryptionPromise = null` below - otherwise we
+                //   risk a race:
+                //
+                //   * A: we check _retryDecryption here and see that it is
+                //        false
+                //   * B: we get a second call to attemptDecryption, which sees
+                //        that _decryptionPromise is set so sets
+                //        _retryDecryption
+                //   * A: we continue below, clear _decryptionPromise, and
+                //        never do the retry.
+                //
+                if (this._retryDecryption) {
+                    // decryption error, but we have a retry queued.
+                    console.log(
+                        `Got error decrypting event (id=${this.getId()}: ` +
+                        `${e}), but retrying`,
+                    );
+                    continue;
+                }
+
+                // decryption error, no retries queued. Warn about the error and
+                // set it to m.bad.encrypted.
+                console.warn(
+                    `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
+                );
+
+                res = this._badEncryptedMessage(e.message);
+            }
+
+            // at this point, we've either successfully decrypted the event, or have given up
+            // (and set res to a 'badEncryptedMessage'). Either way, we can now set the
+            // cleartext of the event and raise Event.decrypted.
+            //
+            // make sure we clear '_decryptionPromise' before sending the 'Event.decrypted' event,
+            // otherwise the app will be confused to see `isBeingDecrypted` still set when
+            // there isn't an `Event.decrypted` on the way.
+            //
+            // see also notes on _retryDecryption above.
+            //
+            this._decryptionPromise = null;
+            this._retryDecryption = false;
+            this._setClearData(res);
+
+            // Before we emit the event, clear the push actions so that they can be recalculated
+            // by relevant code. We do this because the clear event has now changed, making it
+            // so that existing rules can be re-run over the applicable properties. Stuff like
+            // highlighting when the user's name is mentioned rely on this happening. We also want
+            // to set the push actions before emitting so that any notification listeners don't
+            // pick up the wrong contents.
+            this.setPushActions(null);
+
+            this.emit("Event.decrypted", this, err);
+
+            return;
+        }
+    },
+
+    _badEncryptedMessage: function(reason) {
+        return {
+            clearEvent: {
+                type: "m.room.message",
+                content: {
+                    msgtype: "m.bad.encrypted",
+                    body: "** Unable to decrypt: " + reason + " **",
+                },
+            },
+        };
    },

    /**
@@ -284,20 +532,28 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     *
     * @fires module:models/event.MatrixEvent#"Event.decrypted"
     *
-     * @param {Object} clearEvent The plaintext payload for the event
-     *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
-     *
-     * @param {Object=} keysProved Keys owned by the sender of this event.
-     *    See {@link module:models/event.MatrixEvent#getKeysProved}.
-     *
-     * @param {Object=} keysClaimed Keys the sender of this event claims.
-     *    See {@link module:models/event.MatrixEvent#getKeysClaimed}.
+     * @param {module:crypto~EventDecryptionResult} decryptionResult
+     *     the decryption result, including the plaintext and some key info
     */
-    setClearData: function(clearEvent, keysProved, keysClaimed) {
-        this._clearEvent = clearEvent;
-        this._keysProved = keysProved || {};
-        this._keysClaimed = keysClaimed || {};
-        this.emit("Event.decrypted", this);
+    _setClearData: function(decryptionResult) {
+        this._clearEvent = decryptionResult.clearEvent;
+        this._senderCurve25519Key =
+            decryptionResult.senderCurve25519Key || null;
+        this._claimedEd25519Key =
+            decryptionResult.claimedEd25519Key || null;
+        this._forwardingCurve25519KeyChain =
+            decryptionResult.forwardingCurve25519KeyChain || [];
+    },
+
+    /**
+     * Gets the cleartext content for this event. If the event is not encrypted,
+     * or encryption has not been completed, this will return null.
+     *
+     * @returns {Object} The cleartext (decrypted) content for the event
+     */
+    getClearContent: function() {
+        const ev = this._clearEvent;
+        return ev && ev.content ? ev.content : null;
    },

    /**
@@ -309,37 +565,72 @@ utils.extend(module.exports.MatrixEvent.prototype, {
    },

    /**
-     * The curve25519 key that sent this event
+     * The curve25519 key for the device that we think sent this event
+     *
+     * For an Olm-encrypted event, this is inferred directly from the DH
+     * exchange at the start of the session: the curve25519 key is involved in
+     * the DH exchange, so only a device which holds the private part of that
+     * key can establish such a session.
+     *
+     * For a megolm-encrypted event, it is inferred from the Olm message which
+     * established the megolm session
+     *
     * @return {string}
     */
    getSenderKey: function() {
-        return this.getKeysProved().curve25519 || null;
-    },
-
-    /**
-     * The keys that must have been owned by the sender of this encrypted event.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     *
-     * @return {Object<string, string>}
-     */
-    getKeysProved: function() {
-        return this._keysProved;
+        return this._senderCurve25519Key;
    },

    /**
     * The additional keys the sender of this encrypted event claims to possess.
-     * <p>
-     * These don't necessarily have to come from this event itself, but may be
-     * implied by the cryptographic session.
-     * For example megolm messages don't claim keys directly, but instead
-     * inherit a claim from the olm message that established the session.
+     *
+     * Just a wrapper for #getClaimedEd25519Key (q.v.)
     *
     * @return {Object<string, string>}
     */
    getKeysClaimed: function() {
-        return this._keysClaimed;
+        return {
+            ed25519: this._claimedEd25519Key,
+        };
+    },
+
+    /**
+     * Get the ed25519 the sender of this event claims to own.
+     *
+     * For Olm messages, this claim is encoded directly in the plaintext of the
+     * event itself. For megolm messages, it is implied by the m.room_key event
+     * which established the megolm session.
+     *
+     * Until we download the device list of the sender, it's just a claim: the
+     * device list gives a proof that the owner of the curve25519 key used for
+     * this event (and returned by #getSenderKey) also owns the ed25519 key by
+     * signing the public curve25519 key with the ed25519 key.
+     *
+     * In general, applications should not use this method directly, but should
+     * instead use MatrixClient.getEventSenderDeviceInfo.
+     *
+     * @return {string}
+     */
+    getClaimedEd25519Key: function() {
+        return this._claimedEd25519Key;
+    },
+
+    /**
+     * Get the curve25519 keys of the devices which were involved in telling us
+     * about the claimedEd25519Key and sender curve25519 key.
+     *
+     * Normally this will be empty, but in the case of a forwarded megolm
+     * session, the sender keys are sent to us by another device (the forwarding
+     * device), which we need to trust to do this. In that case, the result will
+     * be a list consisting of one entry.
+     *
+     * If the device that sent us the key (A) got it from another device which
+     * it wasn't prepared to vouch for (B), the result will be [A, B]. And so on.
+     *
+     * @return {string[]} base64-encoded curve25519 keys, from oldest to newest.
+     */
+    getForwardingCurve25519KeyChain: function() {
+        return this._forwardingCurve25519KeyChain;
    },

    getUnsigned: function() {
@@ -427,7 +718,6 @@ utils.extend(module.exports.MatrixEvent.prototype, {
        this.event = event;
        // successfully sent.
        this.status = null;
-        this._date = new Date(this.event.origin_server_ts);
     },
 });

@@ -468,4 +758,7 @@ const _REDACT_KEEP_CONTENT_MAP = {
 *
 * @param {module:models/event.MatrixEvent} event
 *    The matrix event which has been decrypted
+ * @param {module:crypto/algorithms/base.DecryptionError?} err
+ *    The error that occured during decryption, or `undefined` if no
+ *    error occured.
 */
@@ -0,0 +1,95 @@
+/*
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module models/group
+ */
+const EventEmitter = require("events").EventEmitter;
+
+const utils = require("../utils");
+
+/**
+ * Construct a new Group.
+ *
+ * @param {string} groupId The ID of this group.
+ *
+ * @prop {string} groupId The ID of this group.
+ * @prop {string} name The human-readable display name for this group.
+ * @prop {string} avatarUrl The mxc URL for this group's avatar.
+ * @prop {string} myMembership The logged in user's membership of this group
+ * @prop {Object} inviter Infomation about the user who invited the logged in user
+ *       to the group, if myMembership is 'invite'.
+ * @prop {string} inviter.userId The user ID of the inviter
+ */
+function Group(groupId) {
+    this.groupId = groupId;
+    this.name = null;
+    this.avatarUrl = null;
+    this.myMembership = null;
+    this.inviter = null;
+}
+utils.inherits(Group, EventEmitter);
+
+Group.prototype.setProfile = function(name, avatarUrl) {
+    if (this.name === name && this.avatarUrl === avatarUrl) return;
+
+    this.name = name || this.groupId;
+    this.avatarUrl = avatarUrl;
+
+    this.emit("Group.profile", this);
+};
+
+Group.prototype.setMyMembership = function(membership) {
+    if (this.myMembership === membership) return;
+
+    this.myMembership = membership;
+
+    this.emit("Group.myMembership", this);
+};
+
+/**
+ * Sets the 'inviter' property. This does not emit an event (the inviter
+ * will only change when the user is revited / reinvited to a room),
+ * so set this before setting myMembership.
+ * @param {Object} inviter Infomation about who invited us to the room
+ */
+Group.prototype.setInviter = function(inviter) {
+    this.inviter = inviter;
+};
+
+module.exports = Group;
+
+/**
+ * Fires whenever a group's profile information is updated.
+ * This means the 'name' and 'avatarUrl' properties.
+ * @event module:client~MatrixClient#"Group.profile"
+ * @param {Group} group The group whose profile was updated.
+ * @example
+ * matrixClient.on("Group.profile", function(group){
+ *   var name = group.name;
+ * });
+ */
+
+/**
+ * Fires whenever the logged in user's membership status of
+ * the group is updated.
+ * @event module:client~MatrixClient#"Group.myMembership"
+ * @param {Group} group The group in which the user's membership changed
+ * @example
+ * matrixClient.on("Group.myMembership", function(group){
+ *   var myMembership = group.myMembership;
+ * });
+ */
@@ -33,7 +33,10 @@ const utils = require("../utils");
 * @prop {string} roomId The room ID for this member.
 * @prop {string} userId The user ID of this member.
 * @prop {boolean} typing True if the room member is currently typing.
- * @prop {string} name The human-readable name for this room member.
+ * @prop {string} name The human-readable name for this room member. This will be
+ * disambiguated with a suffix of " (@user_id:matrix.org)" if another member shares the
+ * same displayname.
+ * @prop {string} rawDisplayName The ambiguous displayname of this room member.
 * @prop {Number} powerLevel The power level for this room member.
 * @prop {Number} powerLevelNorm The normalised power level (0-100) for this
 * room member.
@@ -47,6 +50,7 @@ function RoomMember(roomId, userId) {
    this.userId = userId;
    this.typing = false;
    this.name = userId;
+    this.rawDisplayName = userId;
    this.powerLevel = 0;
    this.powerLevelNorm = 0;
    this.user = null;
@@ -54,10 +58,27 @@ function RoomMember(roomId, userId) {
    this.events = {
        member: null,
    };
+    this._isOutOfBand = false;
    this._updateModifiedTime();
 }
 utils.inherits(RoomMember, EventEmitter);

+/**
+ * Mark the member as coming from a channel that is not sync
+ */
+RoomMember.prototype.markOutOfBand = function() {
+    this._isOutOfBand = true;
+};
+
+/**
+ * @return {bool} does the member come from a channel that is not sync?
+ * This is used to store the member seperately
+ * from the sync state so it available across browser sessions.
+ */
+RoomMember.prototype.isOutOfBand = function() {
+    return this._isOutOfBand;
+};
+
 /**
 * Update this room member's membership event. May fire "RoomMember.name" if
 * this event updates this member's name.
@@ -71,13 +92,21 @@ RoomMember.prototype.setMembershipEvent = function(event, roomState) {
    if (event.getType() !== "m.room.member") {
        return;
    }
+
+    this._isOutOfBand = false;
+
    this.events.member = event;

    const oldMembership = this.membership;
    this.membership = event.getDirectionalContent().membership;

    const oldName = this.name;
-    this.name = calculateDisplayName(this, event, roomState);
+    this.name = calculateDisplayName(
+        this.userId,
+        event.getDirectionalContent().displayname,
+        roomState);
+
+    this.rawDisplayName = event.getDirectionalContent().displayname || this.userId;
    if (oldMembership !== this.membership) {
        this._updateModifiedTime();
        this.emit("RoomMember.membership", event, this, oldMembership);
@@ -99,17 +128,20 @@ RoomMember.prototype.setPowerLevelEvent = function(powerLevelEvent) {
    if (powerLevelEvent.getType() !== "m.room.power_levels") {
        return;
    }
-    let maxLevel = powerLevelEvent.getContent().users_default || 0;
-    utils.forEach(utils.values(powerLevelEvent.getContent().users), function(lvl) {
+
+    const evContent = powerLevelEvent.getDirectionalContent();
+
+    let maxLevel = evContent.users_default || 0;
+    utils.forEach(utils.values(evContent.users), function(lvl) {
        maxLevel = Math.max(maxLevel, lvl);
    });
    const oldPowerLevel = this.powerLevel;
    const oldPowerLevelNorm = this.powerLevelNorm;

-    if (powerLevelEvent.getContent().users[this.userId] !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users[this.userId];
-    } else if (powerLevelEvent.getContent().users_default !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users_default;
+    if (evContent.users && evContent.users[this.userId] !== undefined) {
+        this.powerLevel = evContent.users[this.userId];
+    } else if (evContent.users_default !== undefined) {
+        this.powerLevel = evContent.users_default;
    } else {
        this.powerLevel = 0;
    }
@@ -169,6 +201,44 @@ RoomMember.prototype.getLastModifiedTime = function() {
    return this._modified;
 };

+
+RoomMember.prototype.isKicked = function() {
+    return this.membership === "leave" &&
+        this.events.member.getSender() !== this.events.member.getStateKey();
+};
+
+/**
+ * If this member was invited with the is_direct flag set, return
+ * the user that invited this member
+ * @return {string} user id of the inviter
+ */
+RoomMember.prototype.getDMInviter = function() {
+    // when not available because that room state hasn't been loaded in,
+    // we don't really know, but more likely to not be a direct chat
+    if (this.events.member) {
+        // TODO: persist the is_direct flag on the member as more member events
+        //       come in caused by displayName changes.
+
+        // the is_direct flag is set on the invite member event.
+        // This is copied on the prev_content section of the join member event
+        // when the invite is accepted.
+
+        const memberEvent = this.events.member;
+        let memberContent = memberEvent.getContent();
+        let inviteSender = memberEvent.getSender();
+
+        if (memberContent.membership === "join") {
+            memberContent = memberEvent.getPrevContent();
+            inviteSender = memberEvent.getUnsigned().prev_sender;
+        }
+
+        if (memberContent.membership === "invite" && memberContent.is_direct) {
+            return inviteSender;
+        }
+    }
+};
+
+
 /**
 * Get the avatar URL for a room member.
 * @param {string} baseUrl The base homeserver URL See
@@ -192,10 +262,12 @@ RoomMember.prototype.getAvatarUrl =
    if (allowDefault === undefined) {
        allowDefault = true;
    }
-    if (!this.events.member && !allowDefault) {
+
+    const rawUrl = this.getMxcAvatarUrl();
+
+    if (!rawUrl && !allowDefault) {
        return null;
    }
-    const rawUrl = this.events.member ? this.events.member.getContent().avatar_url : null;
    const httpUrl = ContentRepo.getHttpUriForMxc(
        baseUrl, rawUrl, width, height, resizeMethod, allowDirectLinks,
    );
@@ -208,12 +280,27 @@ RoomMember.prototype.getAvatarUrl =
    }
    return null;
 };
+/**
+ * get the mxc avatar url, either from a state event, or from a lazily loaded member
+ * @return {string} the mxc avatar url
+ */
+RoomMember.prototype.getMxcAvatarUrl = function() {
+    if(this.events.member) {
+        return this.events.member.getDirectionalContent().avatar_url;
+    } else if(this.user) {
+        return this.user.avatarUrl;
+    }
+    return null;
+};

-function calculateDisplayName(member, event, roomState) {
-    const displayName = event.getDirectionalContent().displayname;
-    const selfUserId = member.userId;
+function calculateDisplayName(selfUserId, displayName, roomState) {
+    if (!displayName || displayName === selfUserId) {
+        return selfUserId;
+    }

-    if (!displayName) {
+    // First check if the displayname is something we consider truthy
+    // after stripping it of zero width characters and padding spaces
+    if (!utils.removeHiddenChars(displayName)) {
        return selfUserId;
    }

@@ -221,11 +308,18 @@ function calculateDisplayName(member, event, roomState) {
        return displayName;
    }

-    const userIds = roomState.getUserIdsWithDisplayName(displayName);
-    const otherUsers = userIds.filter(function(u) {
-        return u !== selfUserId;
-    });
-    if (otherUsers.length > 0) {
+    // Next check if the name contains something that look like a mxid
+    // If it does, it may be someone trying to impersonate someone else
+    // Show full mxid in this case
+    // Also show mxid if there are other people with the same or similar
+    // displayname, after hidden character removal.
+    let disambiguate = /@.+:.+/.test(displayName);
+    if (!disambiguate) {
+        const userIds = roomState.getUserIdsWithDisplayName(displayName);
+        disambiguate = userIds.some((u) => u !== selfUserId);
+    }
+
+    if (disambiguate) {
        return displayName + " (" + selfUserId + ")";
    }
    return displayName;
@@ -22,18 +22,46 @@ const EventEmitter = require("events").EventEmitter;
 const utils = require("../utils");
 const RoomMember = require("./room-member");

+// possible statuses for out-of-band member loading
+const OOB_STATUS_NOTSTARTED = 1;
+const OOB_STATUS_INPROGRESS = 2;
+const OOB_STATUS_FINISHED = 3;
+
 /**
 * Construct room state.
+ *
+ * Room State represents the state of the room at a given point.
+ * It can be mutated by adding state events to it.
+ * There are two types of room member associated with a state event:
+ * normal member objects (accessed via getMember/getMembers) which mutate
+ * with the state to represent the current state of that room/user, eg.
+ * the object returned by getMember('@bob:example.com') will mutate to
+ * get a different display name if Bob later changes his display name
+ * in the room.
+ * There are also 'sentinel' members (accessed via getSentinelMember).
+ * These also represent the state of room members at the point in time
+ * represented by the RoomState object, but unlike objects from getMember,
+ * sentinel objects will always represent the room state as at the time
+ * getSentinelMember was called, so if Bob subsequently changes his display
+ * name, a room member object previously acquired with getSentinelMember
+ * will still have his old display name. Calling getSentinelMember again
+ * after the display name change will return a new RoomMember object
+ * with Bob's new display name.
+ *
 * @constructor
 * @param {?string} roomId Optional. The ID of the room which has this state.
 * If none is specified it just tracks paginationTokens, useful for notifTimelineSet
+ * @param {?object} oobMemberFlags Optional. The state of loading out of bound members.
+ * As the timeline might get reset while they are loading, this state needs to be inherited
+ * and shared when the room state is cloned for the new timeline.
+ * This should only be passed from clone.
 * @prop {Object.<string, RoomMember>} members The room member dictionary, keyed
 * on the user's ID.
 * @prop {Object.<string, Object.<string, MatrixEvent>>} events The state
 * events dictionary, keyed on the event type and then the state_key value.
 * @prop {string} paginationToken The pagination token for this state.
 */
-function RoomState(roomId) {
+function RoomState(roomId, oobMemberFlags = undefined) {
    this.roomId = roomId;
    this.members = {
        // userId: RoomMember
@@ -47,12 +75,79 @@ function RoomState(roomId) {
        // userId: RoomMember
    };
    this._updateModifiedTime();
+
+    // stores fuzzy matches to a list of userIDs (applies utils.removeHiddenChars to keys)
    this._displayNameToUserIds = {};
    this._userIdsToDisplayNames = {};
    this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite
+    this._joinedMemberCount = null; // cache of the number of joined members
+    // joined members count from summary api
+    // once set, we know the server supports the summary api
+    // and we should only trust that
+    // we could also only trust that before OOB members
+    // are loaded but doesn't seem worth the hassle atm
+    this._summaryJoinedMemberCount = null;
+    // same for invited member count
+    this._invitedMemberCount = null;
+    this._summaryInvitedMemberCount = null;
+
+    if (!oobMemberFlags) {
+        oobMemberFlags = {
+            status: OOB_STATUS_NOTSTARTED,
+        };
+    }
+    this._oobMemberFlags = oobMemberFlags;
 }
 utils.inherits(RoomState, EventEmitter);

+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+RoomState.prototype.getJoinedMemberCount = function() {
+    if (this._summaryJoinedMemberCount !== null) {
+        return this._summaryJoinedMemberCount;
+    }
+    if (this._joinedMemberCount === null) {
+        this._joinedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'join' ? count + 1 : count;
+        }, 0);
+    }
+    return this._joinedMemberCount;
+};
+
+/**
+ * Set the joined member count explicitly (like from summary part of the sync response)
+ * @param {number} count the amount of joined members
+ */
+RoomState.prototype.setJoinedMemberCount = function(count) {
+    this._summaryJoinedMemberCount = count;
+};
+/**
+ * Returns the number of invited members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite'
+ */
+RoomState.prototype.getInvitedMemberCount = function() {
+    if (this._summaryInvitedMemberCount !== null) {
+        return this._summaryInvitedMemberCount;
+    }
+    if (this._invitedMemberCount === null) {
+        this._invitedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'invite' ? count + 1 : count;
+        }, 0);
+    }
+    return this._invitedMemberCount;
+};
+
+/**
+ * Set the amount of invited members in this room
+ * @param {number} count the amount of invited members
+ */
+RoomState.prototype.setInvitedMemberCount = function(count) {
+    this._summaryInvitedMemberCount = count;
+};
+
 /**
 * Get all RoomMembers in this room.
 * @return {Array<RoomMember>} A list of RoomMembers.
@@ -61,6 +156,16 @@ RoomState.prototype.getMembers = function() {
    return utils.values(this.members);
 };

+/**
+ * Get all RoomMembers in this room, excluding the user IDs provided.
+ * @param {Array<string>} excludedIds The user IDs to exclude.
+ * @return {Array<RoomMember>} A list of RoomMembers.
+ */
+RoomState.prototype.getMembersExcept = function(excludedIds) {
+    return utils.values(this.members)
+        .filter((m) => !excludedIds.includes(m.userId));
+};
+
 /**
 * Get a room member by their user ID.
 * @param {string} userId The room member's user ID.
@@ -80,7 +185,18 @@ RoomState.prototype.getMember = function(userId) {
 * @return {RoomMember} The member or null if they do not exist.
 */
 RoomState.prototype.getSentinelMember = function(userId) {
-    return this._sentinels[userId] || null;
+    if (!userId) return null;
+    let sentinel = this._sentinels[userId];
+
+    if (sentinel === undefined) {
+        sentinel = new RoomMember(this.roomId, userId);
+        const member = this.members[userId];
+        if (member) {
+            sentinel.setMembershipEvent(member.events.member, this);
+        }
+        this._sentinels[userId] = sentinel;
+    }
+    return sentinel;
 };

 /**
@@ -104,6 +220,67 @@ RoomState.prototype.getStateEvents = function(eventType, stateKey) {
    return event ? event : null;
 };

+/**
+ * Creates a copy of this room state so that mutations to either won't affect the other.
+ * @return {RoomState} the copy of the room state
+ */
+RoomState.prototype.clone = function() {
+    const copy = new RoomState(this.roomId, this._oobMemberFlags);
+
+    // Ugly hack: because setStateEvents will mark
+    // members as susperseding future out of bound members
+    // if loading is in progress (through _oobMemberFlags)
+    // since these are not new members, we're merely copying them
+    // set the status to not started
+    // after copying, we set back the status
+    const status = this._oobMemberFlags.status;
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+
+    Object.values(this.events).forEach((eventsByStateKey) => {
+        const eventsForType = Object.values(eventsByStateKey);
+        copy.setStateEvents(eventsForType);
+    });
+
+    // Ugly hack: see above
+    this._oobMemberFlags.status = status;
+
+    if (this._summaryInvitedMemberCount !== null) {
+        copy.setInvitedMemberCount(this.getInvitedMemberCount());
+    }
+    if (this._summaryJoinedMemberCount !== null) {
+        copy.setJoinedMemberCount(this.getJoinedMemberCount());
+    }
+
+    // copy out of band flags if needed
+    if (this._oobMemberFlags.status == OOB_STATUS_FINISHED) {
+        // copy markOutOfBand flags
+        this.getMembers().forEach((member) => {
+            if (member.isOutOfBand()) {
+                const copyMember = copy.getMember(member.userId);
+                copyMember.markOutOfBand();
+            }
+        });
+    }
+
+    return copy;
+};
+
+/**
+ * Add previously unknown state events.
+ * When lazy loading members while back-paginating,
+ * the relevant room state for the timeline chunk at the end
+ * of the chunk can be set with this method.
+ * @param {MatrixEvent[]} events state events to prepend
+ */
+RoomState.prototype.setUnknownStateEvents = function(events) {
+    const unknownStateEvents = events.filter((event) => {
+        return this.events[event.getType()] === undefined ||
+            this.events[event.getType()][event.getStateKey()] === undefined;
+    });
+
+    this.setStateEvents(unknownStateEvents);
+};
+
 /**
 * Add an array of one or more state MatrixEvents, overwriting
 * any existing state with the same {type, stateKey} tuple. Will fire
@@ -127,10 +304,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
            return;
        }

-        if (self.events[event.getType()] === undefined) {
-            self.events[event.getType()] = {};
-        }
-        self.events[event.getType()][event.getStateKey()] = event;
+        self._setStateEvent(event);
        if (event.getType() === "m.room.member") {
            _updateDisplayNameCache(
                self, event.getStateKey(), event.getContent().displayname,
@@ -168,28 +342,10 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                    event.getPrevContent().displayname;
            }

-            let member = self.members[userId];
-            if (!member) {
-                member = new RoomMember(event.getRoomId(), userId);
-                self.emit("RoomState.newMember", event, self, member);
-            }
-            // Add a new sentinel for this change. We apply the same
-            // operations to both sentinel and member rather than deep copying
-            // so we don't make assumptions about the properties of RoomMember
-            // (e.g. and manage to break it because deep copying doesn't do
-            // everything).
-            const sentinel = new RoomMember(event.getRoomId(), userId);
-            utils.forEach([member, sentinel], function(roomMember) {
-                roomMember.setMembershipEvent(event, self);
-                // this member may have a power level already, so set it.
-                const pwrLvlEvent = self.getStateEvents("m.room.power_levels", "");
-                if (pwrLvlEvent) {
-                    roomMember.setPowerLevelEvent(pwrLvlEvent);
-                }
-            });
+            const member = self._getOrCreateMember(userId, event);
+            member.setMembershipEvent(event, self);

-            self._sentinels[userId] = sentinel;
-            self.members[userId] = member;
+            self._updateMember(member);
            self.emit("RoomState.members", event, self, member);
        } else if (event.getType() === "m.room.power_levels") {
            const members = utils.values(self.members);
@@ -197,10 +353,147 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                member.setPowerLevelEvent(event);
                self.emit("RoomState.members", event, self, member);
            });
+
+            // assume all our sentinels are now out-of-date
+            self._sentinels = {};
        }
    });
 };

+/**
+ * Looks up a member by the given userId, and if it doesn't exist,
+ * create it and emit the `RoomState.newMember` event.
+ * This method makes sure the member is added to the members dictionary
+ * before emitting, as this is done from setStateEvents and _setOutOfBandMember.
+ * @param {string} userId the id of the user to look up
+ * @param {MatrixEvent} event the membership event for the (new) member. Used to emit.
+ * @fires module:client~MatrixClient#event:"RoomState.newMember"
+ * @returns {RoomMember} the member, existing or newly created.
+ */
+RoomState.prototype._getOrCreateMember = function(userId, event) {
+    let member = this.members[userId];
+    if (!member) {
+        member = new RoomMember(this.roomId, userId);
+        // add member to members before emitting any events,
+        // as event handlers often lookup the member
+        this.members[userId] = member;
+        this.emit("RoomState.newMember", event, this, member);
+    }
+    return member;
+};
+
+RoomState.prototype._setStateEvent = function(event) {
+    if (this.events[event.getType()] === undefined) {
+        this.events[event.getType()] = {};
+    }
+    this.events[event.getType()][event.getStateKey()] = event;
+};
+
+RoomState.prototype._updateMember = function(member) {
+    // this member may have a power level already, so set it.
+    const pwrLvlEvent = this.getStateEvents("m.room.power_levels", "");
+    if (pwrLvlEvent) {
+        member.setPowerLevelEvent(pwrLvlEvent);
+    }
+
+    // blow away the sentinel which is now outdated
+    delete this._sentinels[member.userId];
+
+    this.members[member.userId] = member;
+    this._joinedMemberCount = null;
+    this._invitedMemberCount = null;
+};
+
+/**
+ * Get the out-of-band members loading state, whether loading is needed or not.
+ * Note that loading might be in progress and hence isn't needed.
+ * @return {bool} whether or not the members of this room need to be loaded
+ */
+RoomState.prototype.needsOutOfBandMembers = function() {
+    return this._oobMemberFlags.status === OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Mark this room state as waiting for out-of-band members,
+ * ensuring it doesn't ask for them to be requested again
+ * through needsOutOfBandMembers
+ */
+RoomState.prototype.markOutOfBandMembersStarted = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_NOTSTARTED) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_INPROGRESS;
+};
+
+/**
+ * Mark this room state as having failed to fetch out-of-band members
+ */
+RoomState.prototype.markOutOfBandMembersFailed = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Clears the loaded out-of-band members
+ */
+RoomState.prototype.clearOutOfBandMembers = function() {
+    let count = 0;
+    Object.keys(this.members).forEach((userId) => {
+        const member = this.members[userId];
+        if (member.isOutOfBand()) {
+            ++count;
+            delete this.members[userId];
+        }
+    });
+    console.log(`LL: RoomState removed ${count} members...`);
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Sets the loaded out-of-band members.
+ * @param {MatrixEvent[]} stateEvents array of membership state events
+ */
+RoomState.prototype.setOutOfBandMembers = function(stateEvents) {
+    console.log(`LL: RoomState about to set ${stateEvents.length} OOB members ...`);
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    console.log(`LL: RoomState put in OOB_STATUS_FINISHED state ...`);
+    this._oobMemberFlags.status = OOB_STATUS_FINISHED;
+    stateEvents.forEach((e) => this._setOutOfBandMember(e));
+};
+
+/**
+ * Sets a single out of band member, used by both setOutOfBandMembers and clone
+ * @param {MatrixEvent} stateEvent membership state event
+ */
+RoomState.prototype._setOutOfBandMember = function(stateEvent) {
+    if (stateEvent.getType() !== 'm.room.member') {
+        return;
+    }
+    const userId = stateEvent.getStateKey();
+    const existingMember = this.getMember(userId);
+    // never replace members received as part of the sync
+    if (existingMember && !existingMember.isOutOfBand()) {
+        return;
+    }
+
+    const member = this._getOrCreateMember(userId, stateEvent);
+    member.setMembershipEvent(stateEvent, this);
+    // needed to know which members need to be stored seperately
+    // as they are not part of the sync accumulator
+    // this is cleared by setMembershipEvent so when it's updated through /sync
+    member.markOutOfBand();
+
+    _updateDisplayNameCache(this, member.userId, member.name);
+
+    this._setStateEvent(stateEvent);
+    this._updateMember(member);
+    this.emit("RoomState.members", stateEvent, this, member);
+};
+
 /**
 * Set the current typing event for this room.
 * @param {MatrixEvent} event The typing event
@@ -238,12 +531,12 @@ RoomState.prototype.getLastModifiedTime = function() {
 };

 /**
- * Get user IDs with the specified display name.
+ * Get user IDs with the specified or similar display names.
 * @param {string} displayName The display name to get user IDs from.
 * @return {string[]} An array of user IDs or an empty array.
 */
 RoomState.prototype.getUserIdsWithDisplayName = function(displayName) {
-    return this._displayNameToUserIds[displayName] || [];
+    return this._displayNameToUserIds[utils.removeHiddenChars(displayName)] || [];
 };

 /**
@@ -258,7 +551,11 @@ RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
    if (!member || member.membership === 'leave') return false;

    if (mxEvent.status || mxEvent.isRedacted()) return false;
-    if (mxEvent.getSender() === userId) return true;
+
+    // The user may have been the sender, but they can't redact their own message
+    // if redactions are blocked.
+    const canRedact = this.maySendEvent("m.room.redaction", userId);
+    if (mxEvent.getSender() === userId) return canRedact;

    return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
 };
@@ -278,7 +575,7 @@ RoomState.prototype._hasSufficientPowerLevelFor = function(action, powerLevel) {
    }

    let requiredLevel = 50;
-    if (powerLevels[action] !== undefined) {
+    if (utils.isNumber(powerLevels[action])) {
        requiredLevel = powerLevels[action];
    }

@@ -350,11 +647,6 @@ RoomState.prototype.maySendStateEvent = function(stateEventType, userId) {
 *                        according to the room's state.
 */
 RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
-    const member = this.getMember(userId);
-    if (!member || member.membership == 'leave') {
-        return false;
-    }
-
    const power_levels_event = this.getStateEvents('m.room.power_levels', '');

    let power_levels;
@@ -362,25 +654,63 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {

    let state_default = 0;
    let events_default = 0;
+    let powerLevel = 0;
    if (power_levels_event) {
        power_levels = power_levels_event.getContent();
        events_levels = power_levels.events || {};

-        if (power_levels.state_default !== undefined) {
+        if (Number.isFinite(power_levels.state_default)) {
            state_default = power_levels.state_default;
        } else {
            state_default = 50;
        }
-        if (power_levels.events_default !== undefined) {
+
+        const userPowerLevel = power_levels.users && power_levels.users[userId];
+        if (Number.isFinite(userPowerLevel)) {
+            powerLevel = userPowerLevel;
+        } else if(Number.isFinite(power_levels.users_default)) {
+            powerLevel = power_levels.users_default;
+        }
+
+        if (Number.isFinite(power_levels.events_default)) {
            events_default = power_levels.events_default;
        }
    }

    let required_level = state ? state_default : events_default;
-    if (events_levels[eventType] !== undefined) {
+    if (Number.isFinite(events_levels[eventType])) {
        required_level = events_levels[eventType];
    }
-    return member.powerLevel >= required_level;
+    return powerLevel >= required_level;
+};
+
+/**
+ * Returns true if the given user ID has permission to trigger notification
+ * of type `notifLevelKey`
+ * @param {string} notifLevelKey The level of notification to test (eg. 'room')
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given user ID has permission to trigger a
+ *                        notification of this type.
+ */
+RoomState.prototype.mayTriggerNotifOfType = function(notifLevelKey, userId) {
+    const member = this.getMember(userId);
+    if (!member) {
+        return false;
+    }
+
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let notifLevel = 50;
+    if (
+        powerLevelsEvent &&
+        powerLevelsEvent.getContent() &&
+        powerLevelsEvent.getContent().notifications &&
+        utils.isNumber(powerLevelsEvent.getContent().notifications[notifLevelKey])
+    ) {
+        notifLevel = powerLevelsEvent.getContent().notifications[notifLevelKey];
+    }
+
+    return member.powerLevel >= notifLevel;
 };

 /**
@@ -414,22 +744,26 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
        // We clobber the user_id > name lookup but the name -> [user_id] lookup
        // means we need to remove that user ID from that array rather than nuking
        // the lot.
-        const existingUserIds = roomState._displayNameToUserIds[oldName] || [];
-        for (let i = 0; i < existingUserIds.length; i++) {
-            if (existingUserIds[i] === userId) {
-                // remove this user ID from this array
-                existingUserIds.splice(i, 1);
-                i--;
-            }
+        const strippedOldName = utils.removeHiddenChars(oldName);
+
+        const existingUserIds = roomState._displayNameToUserIds[strippedOldName];
+        if (existingUserIds) {
+            // remove this user ID from this array
+            const filteredUserIDs = existingUserIds.filter((id) => id !== userId);
+            roomState._displayNameToUserIds[strippedOldName] = filteredUserIDs;
        }
-        roomState._displayNameToUserIds[oldName] = existingUserIds;
    }

    roomState._userIdsToDisplayNames[userId] = displayName;
-    if (!roomState._displayNameToUserIds[displayName]) {
-        roomState._displayNameToUserIds[displayName] = [];
+
+    const strippedDisplayname = displayName && utils.removeHiddenChars(displayName);
+    // an empty stripped displayname (undefined/'') will be set to MXID in room-member.js
+    if (strippedDisplayname) {
+        if (!roomState._displayNameToUserIds[strippedDisplayname]) {
+            roomState._displayNameToUserIds[strippedDisplayname] = [];
+        }
+        roomState._displayNameToUserIds[strippedDisplayname].push(userId);
    }
-    roomState._displayNameToUserIds[displayName].push(userId);
 }

 /**
@@ -459,7 +793,8 @@ function _updateDisplayNameCache(roomState, userId, displayName) {

 /**
 * Fires whenever a member is added to the members dictionary. The RoomMember
- * will not be fully populated yet (e.g. no membership state).
+ * will not be fully populated yet (e.g. no membership state) but will already
+ * be available in the members dictionary.
 * @event module:client~MatrixClient#"RoomState.newMember"
 * @param {MatrixEvent} event The matrix event which caused this event to fire.
 * @param {RoomState} state The room state whose RoomState.members dictionary
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,13 +22,23 @@ const EventEmitter = require("events").EventEmitter;

 const EventStatus = require("./event").EventStatus;
 const RoomSummary = require("./room-summary");
+const RoomMember = require("./room-member");
 const MatrixEvent = require("./event").MatrixEvent;
 const utils = require("../utils");
 const ContentRepo = require("../content-repo");
 const EventTimeline = require("./event-timeline");
 const EventTimelineSet = require("./event-timeline-set");

-import reEmit from '../reemit';
+import ReEmitter from '../ReEmitter';
+
+// These constants are used as sane defaults when the homeserver doesn't support
+// the m.room_versions capability. In practice, KNOWN_SAFE_ROOM_VERSION should be
+// the same as the common default room version whereas SAFE_ROOM_VERSIONS are the
+// room versions which are considered okay for people to run without being asked
+// to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
+// return an m.room_versions capability.
+const KNOWN_SAFE_ROOM_VERSION = '1';
+const SAFE_ROOM_VERSIONS = ['1', '2'];

 function synthesizeReceipt(userId, event, receiptType) {
    // console.log("synthesizing receipt for "+event.getId());
@@ -68,6 +79,8 @@ function synthesizeReceipt(userId, event, receiptType) {
 * @constructor
 * @alias module:models/room
 * @param {string} roomId Required. The ID of this room.
+ * @param {MatrixClient} client Required. The client, used to lazy load members.
+ * @param {string} myUserId Required. The ID of the syncing user.
 * @param {Object=} opts Configuration options
 * @param {*} opts.storageToken Optional. The token which a data store can use
 * to remember the state of the room. What this means is dependent on the store
@@ -94,18 +107,20 @@ function synthesizeReceipt(userId, event, receiptType) {
 * event type and the values are the events.
 * @prop {RoomState} oldState The state of the room at the time of the oldest
 * event in the live timeline. Present for backwards compatibility -
- * prefer getLiveTimeline().getState(true).
+ * prefer getLiveTimeline().getState(EventTimeline.BACKWARDS).
 * @prop {RoomState} currentState The state of the room at the time of the
 * newest event in the timeline. Present for backwards compatibility -
- * prefer getLiveTimeline().getState(false).
+ * prefer getLiveTimeline().getState(EventTimeline.FORWARDS).
 * @prop {RoomSummary} summary The room summary.
 * @prop {*} storageToken A token which a data store can use to remember
 * the state of the room.
 */
-function Room(roomId, opts) {
+function Room(roomId, client, myUserId, opts) {
    opts = opts || {};
    opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";

+    this.reEmitter = new ReEmitter(this);
+
    if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
        throw new Error(
            "opts.pendingEventOrdering MUST be either 'chronological' or " +
@@ -113,6 +128,7 @@ function Room(roomId, opts) {
        );
    }

+    this.myUserId = myUserId;
    this.roomId = roomId;
    this.name = roomId;
    this.tags = {
@@ -153,7 +169,7 @@ function Room(roomId, opts) {
    // all our per-room timeline sets. the first one is the unfiltered ones;
    // the subsequent ones are the filtered ones in no particular order.
    this._timelineSets = [new EventTimelineSet(this, opts)];
-    reEmit(this, this.getUnfilteredTimelineSet(),
+    this.reEmitter.reEmit(this.getUnfilteredTimelineSet(),
           ["Room.timeline", "Room.timelineReset"]);

    this._fixUpLegacyTimelineFields();
@@ -167,10 +183,120 @@ function Room(roomId, opts) {
        this._pendingEventList = [];
    }

-    this._blacklistUnverifiedDevices = false; // read by megolm
+    // read by megolm; boolean value - null indicates "use global value"
+    this._blacklistUnverifiedDevices = null;
+    this._selfMembership = null;
+    this._summaryHeroes = null;
+    // awaited by getEncryptionTargetMembers while room members are loading
+
+    this._client = client;
+    if (!this._opts.lazyLoadMembers) {
+        this._membersPromise = Promise.resolve();
+    } else {
+        this._membersPromise = null;
+    }
 }
+
 utils.inherits(Room, EventEmitter);

+/**
+ * Gets the version of the room
+ * @returns {string} The version of the room, or null if it could not be determined
+ */
+Room.prototype.getVersion = function() {
+    const createEvent = this.currentState.getStateEvents("m.room.create", "");
+    if (!createEvent) {
+        console.warn("Room " + this.room_id + " does not have an m.room.create event");
+        return '1';
+    }
+    const ver = createEvent.getContent()['room_version'];
+    if (ver === undefined) return '1';
+    return ver;
+};
+
+/**
+ * Determines whether this room needs to be upgraded to a new version
+ * @returns {string?} What version the room should be upgraded to, or null if
+ *     the room does not require upgrading at this time.
+ * @deprecated Use #getRecommendedVersion() instead
+ */
+Room.prototype.shouldUpgradeToVersion = function() {
+    // TODO: Remove this function.
+    // This makes assumptions about which versions are safe, and can easily
+    // be wrong. Instead, people are encouraged to use getRecommendedVersion
+    // which determines a safer value. This function doesn't use that function
+    // because this is not async-capable, and to avoid breaking the contract
+    // we're deprecating this.
+
+    if (!SAFE_ROOM_VERSIONS.includes(this.getVersion())) {
+        return KNOWN_SAFE_ROOM_VERSION;
+    }
+
+    return null;
+};
+
+/**
+ * Determines the recommended room version for the room. This returns an
+ * object with 3 properties: <code>version</code> as the new version the
+ * room should be upgraded to (may be the same as the current version);
+ * <code>needsUpgrade</code> to indicate if the room actually can be
+ * upgraded (ie: does the current version not match?); and <code>urgent</code>
+ * to indicate if the new version patches a vulnerability in a previous
+ * version.
+ * @returns {Promise<{version: string, needsUpgrade: bool, urgent: bool}>}
+ * Resolves to the version the room should be upgraded to.
+ */
+Room.prototype.getRecommendedVersion = async function() {
+    const capabilities = await this._client.getCapabilities();
+    let versionCap = capabilities["m.room_versions"];
+    if (!versionCap) {
+        versionCap = {
+            default: KNOWN_SAFE_ROOM_VERSION,
+            available: {},
+        };
+        for (const safeVer of SAFE_ROOM_VERSIONS) {
+            versionCap.available[safeVer] = "stable";
+        }
+    }
+
+    const currentVersion = this.getVersion();
+
+    const result = {
+        version: currentVersion,
+        needsUpgrade: false,
+        urgent: false,
+    };
+
+    // If the room is on the default version then nothing needs to change
+    if (currentVersion === versionCap.default) return Promise.resolve(result);
+
+    const stableVersions = Object.keys(versionCap.available)
+        .filter((v) => versionCap.available[v] === 'stable');
+
+    // Check if the room is on an unstable version. We determine urgency based
+    // off the version being in the Matrix spec namespace or not (if the version
+    // is in the current namespace and unstable, the room is probably vulnerable).
+    if (!stableVersions.includes(currentVersion)) {
+        result.version = versionCap.default;
+        result.needsUpgrade = true;
+        result.urgent = !!this.getVersion().match(/^[0-9]+[0-9.]*$/g);
+        return Promise.resolve(result);
+    }
+
+    // The room is on a stable, but non-default, version by this point.
+    // No upgrade needed.
+    return Promise.resolve(result);
+};
+
+/**
+ * Determines whether the given user is permitted to perform a room upgrade
+ * @param {String} userId The ID of the user to test against
+ * @returns {bool} True if the given user is permitted to upgrade the room
+ */
+Room.prototype.userMayUpgradeRoom = function(userId) {
+    return this.currentState.maySendStateEvent("m.room.tombstone", userId);
+};
+
 /**
 * Get the list of pending sent events for this room
 *
@@ -198,6 +324,232 @@ Room.prototype.getLiveTimeline = function() {
    return this.getUnfilteredTimelineSet().getLiveTimeline();
 };

+/**
+ * @param {string} myUserId the user id for the logged in member
+ * @return {string} the membership type (join | leave | invite) for the logged in user
+ */
+Room.prototype.getMyMembership = function() {
+    return this._selfMembership;
+};
+
+/**
+ * If this room is a DM we're invited to,
+ * try to find out who invited us
+ * @return {string} user id of the inviter
+ */
+Room.prototype.getDMInviter = function() {
+    if (this.myUserId) {
+        const me = this.getMember(this.myUserId);
+        if (me) {
+            return me.getDMInviter();
+        }
+    }
+    if (this._selfMembership === "invite") {
+        // fall back to summary information
+        const memberCount = this.getInvitedAndJoinedMemberCount();
+        if (memberCount == 2 && this._summaryHeroes.length) {
+            return this._summaryHeroes[0];
+        }
+    }
+};
+
+/**
+ * Assuming this room is a DM room, tries to guess with which user.
+ * @return {string} user id of the other member (could be syncing user)
+ */
+Room.prototype.guessDMUserId = function() {
+    const me = this.getMember(this.myUserId);
+    if (me) {
+        const inviterId = me.getDMInviter();
+        if (inviterId) {
+            return inviterId;
+        }
+    }
+    // remember, we're assuming this room is a DM,
+    // so returning the first member we find should be fine
+    const hasHeroes = Array.isArray(this._summaryHeroes) &&
+        this._summaryHeroes.length;
+    if (hasHeroes) {
+        return this._summaryHeroes[0];
+    }
+    const members = this.currentState.getMembers();
+    const anyMember = members.find((m) => m.userId !== this.myUserId);
+    if (anyMember) {
+        return anyMember.userId;
+    }
+    // it really seems like I'm the only user in the room
+    // so I probably created a room with just me in it
+    // and marked it as a DM. Ok then
+    return this.myUserId;
+};
+
+Room.prototype.getAvatarFallbackMember = function() {
+    const memberCount = this.getInvitedAndJoinedMemberCount();
+    if (memberCount > 2) {
+        return;
+    }
+    const hasHeroes = Array.isArray(this._summaryHeroes) &&
+        this._summaryHeroes.length;
+    if (hasHeroes) {
+        const availableMember = this._summaryHeroes.map((userId) => {
+            return this.getMember(userId);
+        }).find((member) => !!member);
+        if (availableMember) {
+            return availableMember;
+        }
+    }
+    const members = this.currentState.getMembers();
+    // could be different than memberCount
+    // as this includes left members
+    if (members.length <= 2) {
+        const availableMember = members.find((m) => {
+            return m.userId !== this.myUserId;
+        });
+        if (availableMember) {
+            return availableMember;
+        }
+    }
+    // if all else fails, try falling back to a user,
+    // and create a one-off member for it
+    if (hasHeroes) {
+        const availableUser = this._summaryHeroes.map((userId) => {
+            return this._client.getUser(userId);
+        }).find((user) => !!user);
+        if (availableUser) {
+            const member = new RoomMember(
+                this.roomId, availableUser.userId);
+            member.user = availableUser;
+            return member;
+        }
+    }
+};
+
+/**
+ * Sets the membership this room was received as during sync
+ * @param {string} membership join | leave | invite
+ */
+Room.prototype.updateMyMembership = function(membership) {
+    const prevMembership = this._selfMembership;
+    this._selfMembership = membership;
+    if (prevMembership !== membership) {
+        if (membership === "leave") {
+            this._cleanupAfterLeaving();
+        }
+        this.emit("Room.myMembership", this, membership, prevMembership);
+    }
+};
+
+Room.prototype._loadMembersFromServer = async function() {
+    const lastSyncToken = this._client.store.getSyncToken();
+    const queryString = utils.encodeParams({
+        not_membership: "leave",
+        at: lastSyncToken,
+    });
+    const path = utils.encodeUri("/rooms/$roomId/members?" + queryString,
+        {$roomId: this.roomId});
+    const http = this._client._http;
+    const response = await http.authedRequest(undefined, "GET", path);
+    return response.chunk;
+};
+
+
+Room.prototype._loadMembers = async function() {
+    // were the members loaded from the server?
+    let fromServer = false;
+    let rawMembersEvents =
+        await this._client.store.getOutOfBandMembers(this.roomId);
+    if (rawMembersEvents === null) {
+        fromServer = true;
+        rawMembersEvents = await this._loadMembersFromServer();
+        console.log(`LL: got ${rawMembersEvents.length} ` +
+            `members from server for room ${this.roomId}`);
+    }
+    const memberEvents = rawMembersEvents.map(this._client.getEventMapper());
+    return {memberEvents, fromServer};
+};
+
+/**
+ * Preloads the member list in case lazy loading
+ * of memberships is in use. Can be called multiple times,
+ * it will only preload once.
+ * @return {Promise} when preloading is done and
+ * accessing the members on the room will take
+ * all members in the room into account
+ */
+Room.prototype.loadMembersIfNeeded = function() {
+    if (this._membersPromise) {
+        return this._membersPromise;
+    }
+
+    // mark the state so that incoming messages while
+    // the request is in flight get marked as superseding
+    // the OOB members
+    this.currentState.markOutOfBandMembersStarted();
+
+    const inMemoryUpdate = this._loadMembers().then((result) => {
+        this.currentState.setOutOfBandMembers(result.memberEvents);
+        // now the members are loaded, start to track the e2e devices if needed
+        if (this._client.isRoomEncrypted(this.roomId)) {
+            this._client._crypto.trackRoomDevices(this.roomId);
+        }
+        return result.fromServer;
+    }).catch((err) => {
+        // allow retries on fail
+        this._membersPromise = null;
+        this.currentState.markOutOfBandMembersFailed();
+        throw err;
+    });
+    // update members in storage, but don't wait for it
+    inMemoryUpdate.then((fromServer) => {
+        if (fromServer) {
+            const oobMembers = this.currentState.getMembers()
+                .filter((m) => m.isOutOfBand())
+                .map((m) => m.events.member.event);
+            console.log(`LL: telling store to write ${oobMembers.length}`
+                + ` members for room ${this.roomId}`);
+            const store = this._client.store;
+            return store.setOutOfBandMembers(this.roomId, oobMembers)
+                // swallow any IDB error as we don't want to fail
+                // because of this
+                .catch((err) => {
+                    console.log("LL: storing OOB room members failed, oh well",
+                        err);
+                });
+        }
+    }).catch((err) => {
+        // as this is not awaited anywhere,
+        // at least show the error in the console
+        console.error(err);
+    });
+
+    this._membersPromise = inMemoryUpdate;
+
+    return this._membersPromise;
+};
+
+/**
+ * Removes the lazily loaded members from storage if needed
+ */
+Room.prototype.clearLoadedMembersIfNeeded = async function() {
+    if (this._opts.lazyLoadMembers && this._membersPromise) {
+        await this.loadMembersIfNeeded();
+        await this._client.store.clearOutOfBandMembers(this.roomId);
+        this.currentState.clearOutOfBandMembers();
+        this._membersPromise = null;
+    }
+};
+
+/**
+ * called when sync receives this room in the leave section
+ * to do cleanup after leaving a room. Possibly called multiple times.
+ */
+Room.prototype._cleanupAfterLeaving = function() {
+    this.clearLoadedMembersIfNeeded().catch((err) => {
+        console.error(`error after clearing loaded members from ` +
+            `room ${this.roomId} after leaving`);
+        console.dir(err);
+    });
+};

 /**
 * Reset the live timeline of all timelineSets, and start new ones.
@@ -205,12 +557,16 @@ Room.prototype.getLiveTimeline = function() {
 * <p>This is used when /sync returns a 'limited' timeline.
 *
 * @param {string=} backPaginationToken   token for back-paginating the new timeline
- * @param {boolean=} flush True to remove all events in all timelines. If false, only
- * the live timeline is reset.
+ * @param {string=} forwardPaginationToken token for forward-paginating the old live timeline,
+ * if absent or null, all timelines are reset, removing old ones (including the previous live
+ * timeline which would otherwise be unable to paginate forwards without this token).
+ * Removing just the old live timeline whilst preserving previous ones is not supported.
 */
-Room.prototype.resetLiveTimeline = function(backPaginationToken, flush) {
+Room.prototype.resetLiveTimeline = function(backPaginationToken, forwardPaginationToken) {
    for (let i = 0; i < this._timelineSets.length; i++) {
-        this._timelineSets[i].resetLiveTimeline(backPaginationToken, flush);
+        this._timelineSets[i].resetLiveTimeline(
+            backPaginationToken, forwardPaginationToken,
+        );
    }

    this._fixUpLegacyTimelineFields();
@@ -233,6 +589,24 @@ Room.prototype._fixUpLegacyTimelineFields = function() {
                            .getState(EventTimeline.FORWARDS);
 };

+/**
+ * Returns whether there are any devices in the room that are unverified
+ * @return {bool} the result
+ */
+Room.prototype.hasUnverifiedDevices = async function() {
+    if (!this._client.isRoomEncrypted(this.roomId)) {
+        return false;
+    }
+    const e2eMembers = await this.getEncryptionTargetMembers();
+    for (const member of e2eMembers) {
+        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        if (devices.some((device) => device.isUnverified())) {
+            return true;
+        }
+    }
+    return false;
+};
+
 /**
 * Return the timeline sets for this room.
 * @return {EventTimelineSet[]} array of timeline sets for this room
@@ -299,10 +673,30 @@ Room.prototype.setUnreadNotificationCount = function(type, count) {
    this._notificationCounts[type] = count;
 };

+Room.prototype.setSummary = function(summary) {
+    const heroes = summary["m.heroes"];
+    const joinedCount = summary["m.joined_member_count"];
+    const invitedCount = summary["m.invited_member_count"];
+    if (Number.isInteger(joinedCount)) {
+        this.currentState.setJoinedMemberCount(joinedCount);
+    }
+    if (Number.isInteger(invitedCount)) {
+        this.currentState.setInvitedMemberCount(invitedCount);
+    }
+    if (Array.isArray(heroes)) {
+        // be cautious about trusting server values,
+        // and make sure heroes doesn't contain our own id
+        // just to be sure
+        this._summaryHeroes = heroes.filter((userId) => {
+            return userId !== this.myUserId;
+        });
+    }
+};
+
 /**
 * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @param {boolean} value if true, blacklist unverified devices.
+ * @param {Boolean} value true to blacklist unverified devices, null
+ * to use the global value for this room.
 */
 Room.prototype.setBlacklistUnverifiedDevices = function(value) {
    this._blacklistUnverifiedDevices = value;
@@ -310,8 +704,8 @@ Room.prototype.setBlacklistUnverifiedDevices = function(value) {

 /**
 * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @return {boolean} true if blacklisting unverified devices.
+ * @return {Boolean} true if blacklisting unverified devices, null
+ * if the global value should be used for this room.
 */
 Room.prototype.getBlacklistUnverifiedDevices = function() {
    return this._blacklistUnverifiedDevices;
@@ -423,11 +817,7 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
 * @return {RoomMember} The member or <code>null</code>.
 */
 Room.prototype.getMember = function(userId) {
-    const member = this.currentState.members[userId];
-    if (!member) {
-        return null;
-    }
-    return member;
+    return this.currentState.getMember(userId);
 };

 /**
@@ -438,6 +828,33 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
    return this.getMembersWithMembership("join");
 };

+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * This is a wrapper around the method of the same name in roomState, returning
+ * its result for the room's current state.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+Room.prototype.getJoinedMemberCount = function() {
+    return this.currentState.getJoinedMemberCount();
+};
+
+/**
+ * Returns the number of invited members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite'
+ */
+Room.prototype.getInvitedMemberCount = function() {
+    return this.currentState.getInvitedMemberCount();
+};
+
+/**
+ * Returns the number of invited + joined members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite' or 'join'
+ */
+Room.prototype.getInvitedAndJoinedMemberCount = function() {
+    return this.getInvitedMemberCount() + this.getJoinedMemberCount();
+};
+
 /**
 * Get a list of members with given membership state.
 * @param {string} membership The membership state.
@@ -449,6 +866,29 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
    });
 };

+ /**
+  * Get a list of members we should be encrypting for in this room
+  * @return {Promise<RoomMember[]>} A list of members who
+  * we should encrypt messages for in this room.
+  */
+ Room.prototype.getEncryptionTargetMembers = async function() {
+    await this.loadMembersIfNeeded();
+    let members = this.getMembersWithMembership("join");
+    if (this.shouldEncryptForInvitedMembers()) {
+        members = members.concat(this.getMembersWithMembership("invite"));
+    }
+    return members;
+ };
+
+ /**
+  * Determine whether we should encrypt messages for invited users in this room
+  * @return {boolean} if we should encrypt messages for invited users
+  */
+ Room.prototype.shouldEncryptForInvitedMembers = function() {
+    const ev = this.currentState.getStateEvents("m.room.history_visibility", "");
+    return (ev && ev.getContent() && ev.getContent().history_visibility !== "joined");
+ };
+
 /**
  * Get the default room name (i.e. what a given user would see if the
  * room had no m.room.name)
@@ -486,7 +926,7 @@ Room.prototype.getOrCreateFilteredTimelineSet = function(filter) {
    }
    const opts = Object.assign({ filter: filter }, this._opts);
    const timelineSet = new EventTimelineSet(this, opts);
-    reEmit(this, timelineSet, ["Room.timeline", "Room.timelineReset"]);
+    this.reEmitter.reEmit(timelineSet, ["Room.timeline", "Room.timelineReset"]);
    this._filteredTimelineSets[filter.filterId] = timelineSet;
    this._timelineSets.push(timelineSet);

@@ -590,7 +1030,8 @@ Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
    // synthesize and inject implicit read receipts
    // Done after adding the event because otherwise the app would get a read receipt
    // pointing to an event that wasn't yet in the timeline
-    if (event.sender) {
+    // Don't synthesize RR for m.room.redaction as this causes the RR to go missing.
+    if (event.sender && event.getType() !== "m.room.redaction") {
        this.addReceipt(synthesizeReceipt(
            event.sender.userId, event, "m.read",
        ), true);
@@ -645,6 +1086,10 @@ Room.prototype.addPendingEvent = function(event, txnId) {
    this._txnToEvent[txnId] = event;

    if (this._opts.pendingEventOrdering == "detached") {
+        if (this._pendingEventList.some((e) => e.status === EventStatus.NOT_SENT)) {
+            console.warn("Setting event as NOT_SENT due to messages in the same state");
+            event.status = EventStatus.NOT_SENT;
+        }
        this._pendingEventList.push(event);
    } else {
        for (let i = 0; i < this._timelineSets.length; i++) {
@@ -809,7 +1254,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
        this.removeEvent(oldEventId);
    }

-    this.emit("Room.localEchoUpdated", event, this, event.getId(), oldStatus);
+    this.emit("Room.localEchoUpdated", event, this, oldEventId, oldStatus);
 };


@@ -901,15 +1346,14 @@ Room.prototype.removeEvent = function(eventId) {
 * Recalculate various aspects of the room, including the room name and
 * room summary. Call this any time the room's current state is modified.
 * May fire "Room.name" if the room name is updated.
- * @param {string} userId The client's user ID.
 * @fires module:client~MatrixClient#event:"Room.name"
 */
-Room.prototype.recalculate = function(userId) {
+Room.prototype.recalculate = function() {
    // set fake stripped state events if this is an invite room so logic remains
    // consistent elsewhere.
    const self = this;
    const membershipEvent = this.currentState.getStateEvents(
-        "m.room.member", userId,
+        "m.room.member", this.myUserId,
    );
    if (membershipEvent && membershipEvent.getContent().membership === "invite") {
        const strippedStateEvents = membershipEvent.event.invite_room_state || [];
@@ -925,14 +1369,14 @@ Room.prototype.recalculate = function(userId) {
                    content: strippedEvent.content,
                    event_id: "$fake" + Date.now(),
                    room_id: self.roomId,
-                    user_id: userId, // technically a lie
+                    user_id: self.myUserId, // technically a lie
                })]);
            }
        });
    }

    const oldName = this.name;
-    this.name = calculateRoomName(this, userId);
+    this.name = calculateRoomName(this, this.myUserId);
    this.summary = new RoomSummary(this.roomId, {
        title: this.name,
    });
@@ -942,7 +1386,6 @@ Room.prototype.recalculate = function(userId) {
    }
 };

-
 /**
 * Get a list of user IDs who have <b>read up to</b> the given event.
 * @param {MatrixEvent} event the event to get read receipts for.
@@ -981,6 +1424,40 @@ Room.prototype.getEventReadUpTo = function(userId, ignoreSynthesized) {
    return receipts["m.read"][userId].eventId;
 };

+/**
+ * Determines if the given user has read a particular event ID with the known
+ * history of the room. This is not a definitive check as it relies only on
+ * what is available to the room at the time of execution.
+ * @param {String} userId The user ID to check the read state of.
+ * @param {String} eventId The event ID to check if the user read.
+ * @returns {Boolean} True if the user has read the event, false otherwise.
+ */
+Room.prototype.hasUserReadEvent = function(userId, eventId) {
+    const readUpToId = this.getEventReadUpTo(userId, false);
+    if (readUpToId === eventId) return true;
+
+    if (this.timeline.length
+        && this.timeline[this.timeline.length - 1].getSender()
+        && this.timeline[this.timeline.length - 1].getSender() === userId) {
+        // It doesn't matter where the event is in the timeline, the user has read
+        // it because they've sent the latest event.
+        return true;
+    }
+
+    for (let i = this.timeline.length - 1; i >= 0; --i) {
+        const ev = this.timeline[i];
+
+        // If we encounter the target event first, the user hasn't read it
+        // however if we encounter the readUpToId first then the user has read
+        // it. These rules apply because we're iterating bottom-up.
+        if (ev.getId() === eventId) return false;
+        if (ev.getId() === readUpToId) return true;
+    }
+
+    // We don't know if the user has read it, so assume not.
+    return false;
+};
+
 /**
 * Get a list of receipts for the given event.
 * @param {MatrixEvent} event the event to get receipts for
@@ -1114,7 +1591,7 @@ Room.prototype.addTags = function(event) {
    // }

    // XXX: do we need to deep copy here?
-    this.tags = event.getContent().tags;
+    this.tags = event.getContent().tags || {};

    // XXX: we could do a deep-comparison to see if the tags have really
    // changed - but do we want to bother?
@@ -1145,6 +1622,17 @@ Room.prototype.getAccountData = function(type) {
    return this.accountData[type];
 };

+
+/**
+ * Returns wheter the syncing user has permission to send a message in the room
+ * @return {boolean} true if the user should be permitted to send
+ *                   message events into the room.
+ */
+Room.prototype.maySendMessage = function() {
+    return this.getMyMembership() === 'join' &&
+        this.currentState.maySendEvent('m.room.message', this.myUserId);
+};
+
 /**
 * This is an internal method. Calculates the name of the room from the current
 * room state.
@@ -1178,80 +1666,83 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
        return alias;
    }

-    // get members that are NOT ourselves and are actually in the room.
-    const otherMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (
-            m.userId !== userId && m.membership !== "leave" && m.membership !== "ban"
-        );
-    });
-    const allMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.membership !== "leave");
-    });
-    const myMemberEventArray = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.userId == userId);
-    });
-    const myMemberEvent = (
-        (myMemberEventArray.length && myMemberEventArray[0].events) ?
-            myMemberEventArray[0].events.member.event : undefined
-    );
+    const joinedMemberCount = room.currentState.getJoinedMemberCount();
+    const invitedMemberCount = room.currentState.getInvitedMemberCount();
+    // -1 because these numbers include the syncing user
+    const inviteJoinCount = joinedMemberCount + invitedMemberCount - 1;

-    // TODO: Localisation
-    if (myMemberEvent && myMemberEvent.content.membership == "invite") {
-        if (room.currentState.getMember(myMemberEvent.sender)) {
-            // extract who invited us to the room
-            return "Invite from " + room.currentState.getMember(
-                myMemberEvent.sender,
-            ).name;
-        } else if (allMembers[0].events.member) {
-            // use the sender field from the invite event, although this only
-            // gets us the mxid
-            return "Invite from " + myMemberEvent.sender;
-        } else {
-            return "Room Invite";
-        }
+    // get members that are NOT ourselves and are actually in the room.
+    let otherNames = null;
+    if (room._summaryHeroes) {
+        // if we have a summary, the member state events
+        // should be in the room state
+        otherNames = room._summaryHeroes.map((userId) => {
+            const member = room.getMember(userId);
+            return member ? member.name : userId;
+        });
+    } else {
+        let otherMembers = room.currentState.getMembers().filter((m) => {
+            return m.userId !== userId &&
+                (m.membership === "invite" || m.membership === "join");
+        });
+        // make sure members have stable order
+        otherMembers.sort((a, b) => a.userId.localeCompare(b.userId));
+        // only 5 first members, immitate _summaryHeroes
+        otherMembers = otherMembers.slice(0, 5);
+        otherNames = otherMembers.map((m) => m.name);
    }

+    if (inviteJoinCount) {
+        return memberNamesToRoomName(otherNames, inviteJoinCount);
+    }

-    if (otherMembers.length === 0) {
-        if (allMembers.length === 1) {
-            // self-chat, peeked room with 1 participant,
-            // or inbound invite, or outbound 3PID invite.
-            if (allMembers[0].userId === userId) {
-                const thirdPartyInvites =
-                    room.currentState.getStateEvents("m.room.third_party_invite");
-                if (thirdPartyInvites && thirdPartyInvites.length > 0) {
-                    let name = "Inviting " +
-                               thirdPartyInvites[0].getContent().display_name;
-                    if (thirdPartyInvites.length > 1) {
-                        if (thirdPartyInvites.length == 2) {
-                            name += " and " +
-                                    thirdPartyInvites[1].getContent().display_name;
-                        } else {
-                            name += " and " +
-                                    thirdPartyInvites.length + " others";
-                        }
-                    }
-                    return name;
-                } else {
-                    return "Empty room";
-                }
-            } else {
-                return allMembers[0].name;
-            }
-        } else {
-            // there really isn't anyone in this room...
-            return "Empty room";
+    const myMembership = room.getMyMembership();
+    // if I have created a room and invited people throuh
+    // 3rd party invites
+    if (myMembership == 'join') {
+        const thirdPartyInvites =
+            room.currentState.getStateEvents("m.room.third_party_invite");
+
+        if (thirdPartyInvites && thirdPartyInvites.length) {
+            const thirdPartyNames = thirdPartyInvites.map((i) => {
+                return i.getContent().display_name;
+            });
+
+            return `Inviting ${memberNamesToRoomName(thirdPartyNames)}`;
        }
-    } else if (otherMembers.length === 1) {
-        return otherMembers[0].name;
-    } else if (otherMembers.length === 2) {
-        return (
-            otherMembers[0].name + " and " + otherMembers[1].name
-        );
+    }
+    // let's try to figure out who was here before
+    let leftNames = otherNames;
+    // if we didn't have heroes, try finding them in the room state
+    if(!leftNames.length) {
+        leftNames = room.currentState.getMembers().filter((m) => {
+            return m.userId !== userId &&
+                m.membership !== "invite" &&
+                m.membership !== "join";
+        }).map((m) => m.name);
+    }
+    if(leftNames.length) {
+        return `Empty room (was ${memberNamesToRoomName(leftNames)})`;
    } else {
-        return (
-            otherMembers[0].name + " and " + (otherMembers.length - 1) + " others"
-        );
+        return "Empty room";
+    }
+}
+
+function memberNamesToRoomName(names, count = (names.length + 1)) {
+    const countWithoutMe = count - 1;
+    if (!names.length) {
+       return "Empty room";
+    } else if (names.length === 1 && countWithoutMe <= 1) {
+        return names[0];
+    } else if (names.length === 2 && countWithoutMe <= 2) {
+        return `${names[0]} and ${names[1]}`;
+    } else {
+        const plural = countWithoutMe > 1;
+        if (plural) {
+            return `${names[0]} and ${countWithoutMe} others`;
+        } else {
+            return `${names[0]} and 1 other`;
+        }
    }
 }

@@ -39,6 +39,9 @@ limitations under the License.
 *                when a user was last active.
 * @prop {Boolean} currentlyActive Whether we should consider lastActiveAgo to be
 *               an approximation and that the user should be seen as active 'now'
+ * @prop {string} _unstable_statusMessage The status message for the user, if known. This is
+ *                different from the presenceStatusMsg in that this is not tied to
+ *                the user's presence, and should be represented differently.
 * @prop {Object} events The events describing this user.
 * @prop {MatrixEvent} events.presence The m.presence event for this user.
 */
@@ -46,6 +49,7 @@ function User(userId) {
    this.userId = userId;
    this.presence = "offline";
    this.presenceStatusMsg = null;
+    this._unstable_statusMessage = "";
    this.displayName = userId;
    this.rawDisplayName = userId;
    this.avatarUrl = null;
@@ -179,6 +183,18 @@ User.prototype.getLastActiveTs = function() {
    return this.lastPresenceTs - this.lastActiveAgo;
 };

+/**
+ * Manually set the user's status message.
+ * @param {MatrixEvent} event The <code>im.vector.user_status</code> event.
+ * @fires module:client~MatrixClient#event:"User._unstable_statusMessage"
+ */
+User.prototype._unstable_updateStatusMessage = function(event) {
+    if (!event.getContent()) this._unstable_statusMessage = "";
+    else this._unstable_statusMessage = event.getContent()["status"];
+    this._updateModifiedTime();
+    this.emit("User._unstable_statusMessage", this);
+};
+
 /**
 * The User class.
 */
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,26 +14,30 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+
+import {escapeRegExp, globToRegexp} from "./utils";
+
 /**
 * @module pushprocessor
 */

+const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
+
 /**
 * Construct a Push Processor.
 * @constructor
 * @param {Object} client The Matrix client object to use
 */
 function PushProcessor(client) {
-    const escapeRegExp = function(string) {
-        return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const cachedGlobToRegex = {
+        // $glob: RegExp,
    };

-    const matchingRuleFromKindSet = function(ev, kindset, device) {
-        const rulekinds_in_order = ['override', 'content', 'room', 'sender', 'underride'];
+    const matchingRuleFromKindSet = (ev, kindset, device) => {
        for (let ruleKindIndex = 0;
-                ruleKindIndex < rulekinds_in_order.length;
+                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                ++ruleKindIndex) {
-            const kind = rulekinds_in_order[ruleKindIndex];
+            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
            const ruleset = kindset[kind];

            for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
@@ -46,7 +51,7 @@ function PushProcessor(client) {
                    continue;
                }

-                if (ruleMatchesEvent(rawrule, ev)) {
+                if (this.ruleMatchesEvent(rawrule, ev)) {
                    rule.kind = kind;
                    return rule;
                }
@@ -73,7 +78,7 @@ function PushProcessor(client) {
                rawrule.conditions.push({
                    'kind': 'event_match',
                    'key': 'room_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                });
                break;
            case 'sender':
@@ -83,7 +88,7 @@ function PushProcessor(client) {
                rawrule.conditions.push({
                    'kind': 'event_match',
                    'key': 'user_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                });
                break;
            case 'content':
@@ -106,27 +111,38 @@ function PushProcessor(client) {
        return rawrule;
    };

-    const ruleMatchesEvent = function(rule, ev) {
-        let ret = true;
-        for (let i = 0; i < rule.conditions.length; ++i) {
-            const cond = rule.conditions[i];
-            ret &= eventFulfillsCondition(cond, ev);
-        }
-        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
-        return ret;
-    };
-
    const eventFulfillsCondition = function(cond, ev) {
        const condition_functions = {
            "event_match": eventFulfillsEventMatchCondition,
            "device": eventFulfillsDeviceCondition,
            "contains_display_name": eventFulfillsDisplayNameCondition,
            "room_member_count": eventFulfillsRoomMemberCountCondition,
+            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
        };
        if (condition_functions[cond.kind]) {
            return condition_functions[cond.kind](cond, ev);
        }
-        return true;
+        // unknown conditions: we previously matched all unknown conditions,
+        // but given that rules can be added to the base rules on a server,
+        // it's probably better to not match unknown conditions.
+        return false;
+    };
+
+    const eventFulfillsSenderNotifPermCondition = function(cond, ev) {
+        const notifLevelKey = cond['key'];
+        if (!notifLevelKey) {
+            return false;
+        }
+
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState) {
+            return false;
+        }
+
+        // Note that this should not be the current state of the room but the state at
+        // the point the event is in the DAG. Unfortunately the js-sdk does not store
+        // this.
+        return room.currentState.mayTriggerNotifOfType(notifLevelKey, ev.getSender());
    };

    const eventFulfillsRoomMemberCountCondition = function(cond, ev) {
@@ -139,9 +155,7 @@ function PushProcessor(client) {
            return false;
        }

-        const memberCount = Object.keys(room.currentState.members).filter(function(m) {
-            return room.currentState.members[m].membership == 'join';
-        }).length;
+        const memberCount = room.currentState.getJoinedMemberCount();

        const m = cond.is.match(/^([=<>]*)([0-9]*)$/);
        if (!m) {
@@ -170,7 +184,10 @@ function PushProcessor(client) {
    };

    const eventFulfillsDisplayNameCondition = function(cond, ev) {
-        const content = ev.getContent();
+        let content = ev.getContent();
+        if (ev.isEncrypted() && ev.getClearContent()) {
+            content = ev.getClearContent();
+        }
        if (!content || !content.body || typeof content.body != 'string') {
            return false;
        }
@@ -194,35 +211,39 @@ function PushProcessor(client) {
    };

    const eventFulfillsEventMatchCondition = function(cond, ev) {
+        if (!cond.key) {
+            return false;
+        }
+
        const val = valueForDottedKey(cond.key, ev);
        if (!val || typeof val != 'string') {
            return false;
        }

-        let pat;
-        if (cond.key == 'content.body') {
-            pat = '(^|\\W)' + globToRegexp(cond.pattern) + '(\\W|$)';
-        } else {
-            pat = '^' + globToRegexp(cond.pattern) + '$';
+        if (cond.value) {
+            return cond.value === val;
        }
-        const regex = new RegExp(pat, 'i');
+
+        let regex;
+
+        if (cond.key == 'content.body') {
+            regex = createCachedRegex('(^|\\W)', cond.pattern, '(\\W|$)');
+        } else {
+            regex = createCachedRegex('^', cond.pattern, '$');
+        }
+
        return !!val.match(regex);
    };

-    const globToRegexp = function(glob) {
-        // From
-        // https://github.com/matrix-org/synapse/blob/abbee6b29be80a77e05730707602f3bbfc3f38cb/synapse/push/__init__.py#L132
-        // Because micromatch is about 130KB with dependencies,
-        // and minimatch is not much better.
-        let pat = escapeRegExp(glob);
-        pat = pat.replace(/\\\*/, '.*');
-        pat = pat.replace(/\?/, '.');
-        pat = pat.replace(/\\\[(!|)(.*)\\]/, function(match, p1, p2, offset, string) {
-            const first = p1 && '^' || '';
-            const second = p2.replace(/\\\-/, '-');
-            return '[' + first + second + ']';
-        });
-        return pat;
+    const createCachedRegex = function(prefix, glob, suffix) {
+        if (cachedGlobToRegex[glob]) {
+            return cachedGlobToRegex[glob];
+        }
+        cachedGlobToRegex[glob] = new RegExp(
+            prefix + globToRegexp(glob) + suffix,
+            'i', // Case insensitive
+        );
+        return cachedGlobToRegex[glob];
    };

    const valueForDottedKey = function(key, ev) {
@@ -291,6 +312,17 @@ function PushProcessor(client) {
        return actionObj;
    };

+    this.ruleMatchesEvent = function(rule, ev) {
+        let ret = true;
+        for (let i = 0; i < rule.conditions.length; ++i) {
+            const cond = rule.conditions[i];
+            ret &= eventFulfillsCondition(cond, ev);
+        }
+        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
+        return ret;
+    };
+
+
    /**
     * Get the user's push actions for the given event
     *
@@ -301,6 +333,27 @@ function PushProcessor(client) {
    this.actionsForEvent = function(ev) {
        return pushActionsForEventAndRulesets(ev, client.pushRules);
    };
+
+    /**
+     * Get one of the users push rules by its ID
+     *
+     * @param {string} ruleId The ID of the rule to search for
+     * @return {object} The push rule, or null if no such rule was found
+     */
+    this.getPushRuleById = function(ruleId) {
+        for (const scope of ['device', 'global']) {
+            if (client.pushRules[scope] === undefined) continue;
+
+            for (const kind of RULEKINDS_IN_ORDER) {
+                if (client.pushRules[scope][kind] === undefined) continue;
+
+                for (const rule of client.pushRules[scope][kind]) {
+                    if (rule.rule_id === ruleId) return rule;
+                }
+            }
+        }
+        return null;
+    };
 }

 /**
@@ -0,0 +1,26 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function randomString(len) {
+    let ret = "";
+    const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+    for (let i = 0; i < len; ++i) {
+        ret += chars.charAt(Math.floor(Math.random() * chars.length));
+    }
+
+    return ret;
+}
@@ -1,44 +0,0 @@
-/*
-Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * @module
- */
-
-/**
- * re-emit events raised by one EventEmitter from another
- *
- * @param {external:EventEmitter} reEmitEntity
- *     entity from which we want events to be emitted
- * @param {external:EventEmitter} emittableEntity
- *     entity from which events are currently emitted
- * @param {Array<string>} eventNames
- *     list of events to be reemitted
- */
-export default function reEmit(reEmitEntity, emittableEntity, eventNames) {
-    for (const eventName of eventNames) {
-        // setup a listener on the entity (the Room, User, etc) for this event
-        emittableEntity.on(eventName, function(...args) {
-            // take the args from the listener and reuse them, adding the
-            // event name to the arg list so it works with .emit()
-            // Transformation Example:
-            // listener on "foo" => function(a,b) { ... }
-            // Re-emit on "thing" => thing.emit("foo", a, b)
-            reEmitEntity.emit(eventName, ...args);
-        });
-    }
-}
@@ -20,7 +20,7 @@ limitations under the License.
 * @module scheduler
 */
 const utils = require("./utils");
-const q = require("q");
+import Promise from 'bluebird';

 const DEBUG = false;  // set true to enable console logging.

@@ -83,6 +83,8 @@ MatrixScheduler.prototype.removeEventFromQueue = function(event) {
    let removed = false;
    utils.removeElement(this._queues[name], function(element) {
        if (element.event.getId() === event.getId()) {
+            // XXX we should probably reject the promise?
+            // https://github.com/matrix-org/matrix-js-sdk/issues/496
            removed = true;
            return true;
        }
@@ -118,7 +120,7 @@ MatrixScheduler.prototype.queueEvent = function(event) {
    if (!this._queues[queueName]) {
        this._queues[queueName] = [];
    }
-    const defer = q.defer();
+    const defer = Promise.defer();
    this._queues[queueName].push({
        event: event,
        defer: defer,
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,11 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from "q";
+import Promise from 'bluebird';
 import SyncAccumulator from "../sync-accumulator";
 import utils from "../utils";

-const VERSION = 1;
+const VERSION = 3;

 function createDatabase(db) {
    // Make user store, clobber based on user ID. (userId property of User objects)
@@ -32,6 +33,20 @@ function createDatabase(db) {
    db.createObjectStore("sync", { keyPath: ["clobber"] });
 }

+function upgradeSchemaV2(db) {
+    const oobMembersStore = db.createObjectStore(
+        "oob_membership_events", {
+            keyPath: ["room_id", "state_key"],
+        });
+    oobMembersStore.createIndex("room", "room_id");
+}
+
+function upgradeSchemaV3(db) {
+    db.createObjectStore("client_options",
+        { keyPath: ["clobber"]});
+}
+
+
 /**
 * Helper method to collect results from a Cursor and promiseify it.
 * @param {ObjectStore|Index} store The store to perform openCursor on.
@@ -44,7 +59,7 @@ function createDatabase(db) {
 */
 function selectQuery(store, keyRange, resultMapper) {
    const query = store.openCursor(keyRange);
-    return q.Promise((resolve, reject) => { /*eslint new-cap: 0*/
+    return new Promise((resolve, reject) => {
        const results = [];
        query.onerror = (event) => {
            reject(new Error("Query failed: " + event.target.errorCode));
@@ -62,28 +77,39 @@ function selectQuery(store, keyRange, resultMapper) {
    });
 }

-function promiseifyTxn(txn) {
-    return new q.Promise((resolve, reject) => {
+function txnAsPromise(txn) {
+    return new Promise((resolve, reject) => {
        txn.oncomplete = function(event) {
            resolve(event);
        };
        txn.onerror = function(event) {
-            reject(event);
+            reject(event.target.error);
        };
    });
 }

-function promiseifyRequest(req) {
-    return new q.Promise((resolve, reject) => {
+function reqAsEventPromise(req) {
+    return new Promise((resolve, reject) => {
        req.onsuccess = function(event) {
            resolve(event);
        };
        req.onerror = function(event) {
-            reject(event);
+            reject(event.target.error);
        };
    });
 }

+function reqAsPromise(req) {
+    return new Promise((resolve, reject) => {
+        req.onsuccess = () => resolve(req);
+        req.onerror = (err) => reject(err);
+    });
+}
+
+function reqAsCursorPromise(req) {
+    return reqAsEventPromise(req).then((event) => event.target.result);
+}
+
 /**
 * Does the actual reading from and writing to the indexeddb
 *
@@ -101,7 +127,9 @@ const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
    this.indexedDB = indexedDBInterface;
    this._dbName = "matrix-js-sdk:" + (dbName || "default");
    this.db = null;
+    this._disconnected = true;
    this._syncAccumulator = new SyncAccumulator();
+    this._isNewlyCreated = false;
 };


@@ -112,20 +140,51 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if successfully connected.
     */
    connect: function() {
-        if (this.db) {
-            return q();
+        if (!this._disconnected) {
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: already connected or connecting`,
+            );
+            return Promise.resolve();
        }
+
+        this._disconnected = false;
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: connecting...`,
+        );
        const req = this.indexedDB.open(this._dbName, VERSION);
        req.onupgradeneeded = (ev) => {
            const db = ev.target.result;
            const oldVersion = ev.oldVersion;
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: upgrading from ${oldVersion}`,
+            );
            if (oldVersion < 1) { // The database did not previously exist.
+                this._isNewlyCreated = true;
                createDatabase(db);
            }
+            if (oldVersion < 2) {
+                upgradeSchemaV2(db);
+            }
+            if (oldVersion < 3) {
+                upgradeSchemaV3(db);
+            }
            // Expand as needed.
        };

-        return promiseifyRequest(req).then((ev) => {
+        req.onblocked = () => {
+            console.log(
+                `can't yet open LocalIndexedDBStoreBackend because it is open elsewhere`,
+            );
+        };
+
+        console.log(
+            `LocalIndexedDBStoreBackend.connect: awaiting connection...`,
+        );
+        return reqAsEventPromise(req).then((ev) => {
+            console.log(
+                `LocalIndexedDBStoreBackend.connect: connected`,
+            );
            this.db = ev.target.result;

            // add a poorly-named listener for when deleteDatabase is called
@@ -137,19 +196,27 @@ LocalIndexedDBStoreBackend.prototype = {
            return this._init();
        });
    },
+    /** @return {bool} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(this._isNewlyCreated);
+    },

    /**
     * Having connected, load initial data from the database and prepare for use
     * @return {Promise} Resolves on success
     */
    _init: function() {
-        return q.all([
+        return Promise.all([
            this._loadAccountData(),
            this._loadSyncData(),
        ]).then(([accountData, syncData]) => {
+            console.log(
+                `LocalIndexedDBStoreBackend: loaded initial data`,
+            );
            this._syncAccumulator.accumulate({
                next_batch: syncData.nextBatch,
                rooms: syncData.roomsData,
+                groups: syncData.groupsData,
                account_data: {
                    events: accountData,
                },
@@ -157,19 +224,161 @@ LocalIndexedDBStoreBackend.prototype = {
        });
    },

+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {Promise<event[]>} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return new Promise((resolve, reject) =>{
+            const tx = this.db.transaction(["oob_membership_events"], "readonly");
+            const store = tx.objectStore("oob_membership_events");
+            const roomIndex = store.index("room");
+            const range = IDBKeyRange.only(roomId);
+            const request = roomIndex.openCursor(range);
+
+            const membershipEvents = [];
+            // did we encounter the oob_written marker object
+            // amongst the results? That means OOB member
+            // loading already happened for this room
+            // but there were no members to persist as they
+            // were all known already
+            let oobWritten = false;
+
+            request.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (!cursor) {
+                    // Unknown room
+                    if (!membershipEvents.length && !oobWritten) {
+                        return resolve(null);
+                    }
+                    return resolve(membershipEvents);
+                }
+                const record = cursor.value;
+                if (record.oob_written) {
+                    oobWritten = true;
+                } else {
+                    membershipEvents.push(record);
+                }
+                cursor.continue();
+            };
+            request.onerror = (err) => {
+                reject(err);
+            };
+        }).then((events) => {
+            console.log(`LL: got ${events && events.length}` +
+                ` membershipEvents from storage for room ${roomId} ...`);
+            return events;
+        });
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     */
+    setOutOfBandMembers: async function(roomId, membershipEvents) {
+        console.log(`LL: backend about to store ${membershipEvents.length}` +
+            ` members for ${roomId}`);
+        const tx = this.db.transaction(["oob_membership_events"], "readwrite");
+        const store = tx.objectStore("oob_membership_events");
+        membershipEvents.forEach((e) => {
+            store.put(e);
+        });
+        // aside from all the events, we also write a marker object to the store
+        // to mark the fact that OOB members have been written for this room.
+        // It's possible that 0 members need to be written as all where previously know
+        // but we still need to know whether to return null or [] from getOutOfBandMembers
+        // where null means out of band members haven't been stored yet for this room
+        const markerObject = {
+            room_id: roomId,
+            oob_written: true,
+            state_key: 0,
+        };
+        store.put(markerObject);
+        await txnAsPromise(tx);
+        console.log(`LL: backend done storing for ${roomId}!`);
+    },
+
+    clearOutOfBandMembers: async function(roomId) {
+        // the approach to delete all members for a room
+        // is to get the min and max state key from the index
+        // for that room, and then delete between those
+        // keys in the store.
+        // this should be way faster than deleting every member
+        // individually for a large room.
+        const readTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readonly");
+        const store = readTx.objectStore("oob_membership_events");
+        const roomIndex = store.index("room");
+        const roomRange = IDBKeyRange.only(roomId);
+
+        const minStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "next"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const maxStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "prev"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const [minStateKey, maxStateKey] = await Promise.all(
+            [minStateKeyProm, maxStateKeyProm]);
+
+        const writeTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readwrite");
+        const writeStore = writeTx.objectStore("oob_membership_events");
+        const membersKeyRange = IDBKeyRange.bound(
+            [roomId, minStateKey],
+            [roomId, maxStateKey],
+        );
+
+        console.log(`LL: Deleting all users + marker in storage for ` +
+            `room ${roomId}, with key range:`,
+            [roomId, minStateKey], [roomId, maxStateKey]);
+        await reqAsPromise(writeStore.delete(membersKeyRange));
+    },
+
    /**
     * Clear the entire database. This should be used when logging out of a client
     * to prevent mixing data between accounts.
     * @return {Promise} Resolved when the database is cleared.
     */
    clearDatabase: function() {
-        console.log("Removing indexeddb instance: ", this._dbName);
-        return promiseifyRequest(this.indexedDB.deleteDatabase(this._dbName));
+        return new Promise((resolve, reject) => {
+            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            const req = this.indexedDB.deleteDatabase(this._dbName);
+
+            req.onblocked = () => {
+                console.log(
+                    `can't yet delete indexeddb ${this._dbName}` +
+                    ` because it is open elsewhere`,
+                );
+            };
+
+            req.onerror = (ev) => {
+                // in firefox, with indexedDB disabled, this fails with a
+                // DOMError. We treat this as non-fatal, so that we can still
+                // use the app.
+                console.warn(
+                    `unable to delete js-sdk store indexeddb: ${ev.target.error}`,
+                );
+                resolve();
+            };
+
+            req.onsuccess = () => {
+                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                resolve();
+            };
+        });
    },

    /**
     * @param {boolean=} copy If false, the data returned is from internal
-     * buffers and must not be muated. Otherwise, a copy is made before
+     * buffers and must not be mutated. Otherwise, a copy is made before
     * returning such that the data can be safely mutated. Default: true.
     *
     * @return {Promise} Resolves with a sync response to restore the
@@ -180,18 +389,22 @@ LocalIndexedDBStoreBackend.prototype = {
        if (copy === undefined) copy = true;

        const data = this._syncAccumulator.getJSON();
-        if (!data.nextBatch) return q(null);
+        if (!data.nextBatch) return Promise.resolve(null);
        if (copy) {
            // We must deep copy the stored data so that the /sync processing code doesn't
            // corrupt the internal state of the sync accumulator (it adds non-clonable keys)
-            return q(utils.deepCopy(data));
+            return Promise.resolve(utils.deepCopy(data));
        } else {
-            return q(data);
+            return Promise.resolve(data);
        }
    },

+    getNextBatchToken: function() {
+        return Promise.resolve(this._syncAccumulator.getNextBatchToken());
+    },
+
    setSyncData: function(syncData) {
-        return q().then(() => {
+        return Promise.resolve().then(() => {
            this._syncAccumulator.accumulate(syncData);
        });
    },
@@ -199,10 +412,12 @@ LocalIndexedDBStoreBackend.prototype = {
    syncToDatabase: function(userTuples) {
        const syncData = this._syncAccumulator.getJSON();

-        return q.all([
+        return Promise.all([
            this._persistUserPresenceEvents(userTuples),
            this._persistAccountData(syncData.accountData),
-            this._persistSyncData(syncData.nextBatch, syncData.roomsData),
+            this._persistSyncData(
+                syncData.nextBatch, syncData.roomsData, syncData.groupsData,
+            ),
        ]);
    },

@@ -210,19 +425,21 @@ LocalIndexedDBStoreBackend.prototype = {
     * Persist rooms /sync data along with the next batch token.
     * @param {string} nextBatch The next_batch /sync value.
     * @param {Object} roomsData The 'rooms' /sync data from a SyncAccumulator
+     * @param {Object} groupsData The 'groups' /sync data from a SyncAccumulator
     * @return {Promise} Resolves if the data was persisted.
     */
-    _persistSyncData: function(nextBatch, roomsData) {
+    _persistSyncData: function(nextBatch, roomsData, groupsData) {
        console.log("Persisting sync data up to ", nextBatch);
-        return q.try(() => {
+        return Promise.try(() => {
            const txn = this.db.transaction(["sync"], "readwrite");
            const store = txn.objectStore("sync");
            store.put({
                clobber: "-", // constant key so will always clobber
                nextBatch: nextBatch,
                roomsData: roomsData,
+                groupsData: groupsData,
            }); // put == UPSERT
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -233,13 +450,13 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if the events were persisted.
     */
    _persistAccountData: function(accountData) {
-        return q.try(() => {
+        return Promise.try(() => {
            const txn = this.db.transaction(["accountData"], "readwrite");
            const store = txn.objectStore("accountData");
            for (let i = 0; i < accountData.length; i++) {
                store.put(accountData[i]); // put == UPSERT
            }
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -252,7 +469,7 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if the users were persisted.
     */
    _persistUserPresenceEvents: function(tuples) {
-        return q.try(() => {
+        return Promise.try(() => {
            const txn = this.db.transaction(["users"], "readwrite");
            const store = txn.objectStore("users");
            for (const tuple of tuples) {
@@ -261,7 +478,7 @@ LocalIndexedDBStoreBackend.prototype = {
                    event: tuple[1],
                }); // put == UPSERT
            }
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -272,7 +489,7 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise<Object[]>} A list of presence events in their raw form.
     */
    getUserPresenceEvents: function() {
-        return q.try(() => {
+        return Promise.try(() => {
            const txn = this.db.transaction(["users"], "readonly");
            const store = txn.objectStore("users");
            return selectQuery(store, undefined, (cursor) => {
@@ -286,11 +503,19 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise<Object[]>} A list of raw global account events.
     */
    _loadAccountData: function() {
-        return q.try(() => {
+        console.log(
+            `LocalIndexedDBStoreBackend: loading account data...`,
+        );
+        return Promise.try(() => {
            const txn = this.db.transaction(["accountData"], "readonly");
            const store = txn.objectStore("accountData");
            return selectQuery(store, undefined, (cursor) => {
                return cursor.value;
+            }).then((result) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded account data`,
+                );
+                return result;
            });
        });
    },
@@ -300,12 +525,18 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise<Object>} An object with "roomsData" and "nextBatch" keys.
     */
    _loadSyncData: function() {
-        return q.try(() => {
+        console.log(
+            `LocalIndexedDBStoreBackend: loading sync data...`,
+        );
+        return Promise.try(() => {
            const txn = this.db.transaction(["sync"], "readonly");
            const store = txn.objectStore("sync");
            return selectQuery(store, undefined, (cursor) => {
                return cursor.value;
            }).then((results) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded sync data`,
+                );
                if (results.length > 1) {
                    console.warn("loadSyncData: More than 1 sync row found.");
                }
@@ -313,6 +544,28 @@ LocalIndexedDBStoreBackend.prototype = {
            });
        });
    },
+
+    getClientOptions: function() {
+        return Promise.resolve().then(() => {
+            const txn = this.db.transaction(["client_options"], "readonly");
+            const store = txn.objectStore("client_options");
+            return selectQuery(store, undefined, (cursor) => {
+                if (cursor.value && cursor.value && cursor.value.options) {
+                    return cursor.value.options;
+                }
+            }).then((results) => results[0]);
+        });
+    },
+
+    storeClientOptions: async function(options) {
+        const txn = this.db.transaction(["client_options"], "readwrite");
+        const store = txn.objectStore("client_options");
+        store.put({
+            clobber: "-", // constant key so will always clobber
+            options: options,
+        }); // put == UPSERT
+        await txnAsPromise(txn);
+    },
 };

 export default LocalIndexedDBStoreBackend;
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from "q";
+import Promise from 'bluebird';

 /**
 * An IndexedDB store backend where the actual backend sits in a web
@@ -26,20 +27,23 @@ import q from "q";
 * @param {string} workerScript URL to the worker script
 * @param {string=} dbName Optional database name. The same name must be used
 * to open the same database.
- * @param {Object} WorkerApi The web worker compatible interface object
+ * @param {Object} workerApi The web worker compatible interface object
 */
 const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
-    workerScript, dbName, WorkerApi,
+    workerScript, dbName, workerApi,
 ) {
+    this._workerScript = workerScript;
    this._dbName = dbName;
-    this._worker = new WorkerApi(workerScript);
+    this._workerApi = workerApi;
+    this._worker = null;
    this._nextSeq = 0;
    // The currently in-flight requests to the actual backend
    this._inFlight = {
        // seq: promise,
    };
-
-    this._worker.onmessage = this._onWorkerMessage.bind(this);
+    // Once we start connecting, we keep the promise and re-use it
+    // if we try to connect again
+    this._startPromise = null;
 };


@@ -50,10 +54,7 @@ RemoteIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if successfully connected.
     */
    connect: function() {
-        return this._doCmd('_setupWorker', [this._dbName]).then(() => {
-            console.log("IndexedDB worker is ready");
-            return this._doCmd('connect');
-        });
+        return this._ensureStarted().then(() => this._doCmd('connect'));
    },

    /**
@@ -62,9 +63,12 @@ RemoteIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolved when the database is cleared.
     */
    clearDatabase: function() {
-        return this._doCmd('clearDatabase');
+        return this._ensureStarted().then(() => this._doCmd('clearDatabase'));
+    },
+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return this._doCmd('isNewlyCreated');
    },
-
    /**
     * @return {Promise} Resolves with a sync response to restore the
     * client state to where it was at the last save, or null if there
@@ -74,6 +78,10 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('getSavedSync');
    },

+    getNextBatchToken: function() {
+        return this._doCmd('getNextBatchToken');
+    },
+
    setSyncData: function(syncData) {
        return this._doCmd('setSyncData', [syncData]);
    },
@@ -82,6 +90,40 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('syncToDatabase', [users]);
    },

+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return this._doCmd('getOutOfBandMembers', [roomId]);
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        return this._doCmd('setOutOfBandMembers', [roomId, membershipEvents]);
+    },
+
+    clearOutOfBandMembers: function(roomId) {
+        return this._doCmd('clearOutOfBandMembers', [roomId]);
+    },
+
+    getClientOptions: function() {
+        return this._doCmd('getClientOptions');
+    },
+
+    storeClientOptions: function(options) {
+        return this._doCmd('storeClientOptions', [options]);
+    },

    /**
     * Load all user presence events from the database. This is not cached.
@@ -91,12 +133,25 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('getUserPresenceEvents');
    },

+    _ensureStarted: function() {
+        if (this._startPromise === null) {
+            this._worker = new this._workerApi(this._workerScript);
+            this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+            // tell the worker the db name.
+            this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
+                console.log("IndexedDB worker is ready");
+            });
+        }
+        return this._startPromise;
+    },
+
    _doCmd: function(cmd, args) {
        // wrap in a q so if the postMessage throws,
        // the promise automatically gets rejected
-        return q().then(() => {
+        return Promise.resolve().then(() => {
            const seq = this._nextSeq++;
-            const def = q.defer();
+            const def = Promise.defer();

            this._inFlight[seq] = def;

@@ -129,7 +184,9 @@ RemoteIndexedDBStoreBackend.prototype = {
            if (msg.command == 'cmd_success') {
                def.resolve(msg.result);
            } else {
-                def.reject(msg.error);
+                const error = new Error(msg.error.message);
+                error.name = msg.error.name;
+                def.reject(error);
            }
        } else {
            console.warn("Unrecognised message from worker: " + msg);
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from "q";
+import Promise from 'bluebird';
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";

 /**
@@ -61,11 +62,14 @@ class IndexedDBStoreWorker {
                    // because it's a web worker and there is no window).
                    indexedDB, msg.args[0],
                );
-                prom = q();
+                prom = Promise.resolve();
                break;
            case 'connect':
                prom = this.backend.connect();
                break;
+            case 'isNewlyCreated':
+                prom = this.backend.isNewlyCreated();
+                break;
            case 'clearDatabase':
                prom = this.backend.clearDatabase().then((result) => {
                    // This returns special classes which can't be cloned
@@ -88,10 +92,28 @@ class IndexedDBStoreWorker {
            case 'getUserPresenceEvents':
                prom = this.backend.getUserPresenceEvents();
                break;
+            case 'getNextBatchToken':
+                prom = this.backend.getNextBatchToken();
+                break;
+            case 'getOutOfBandMembers':
+                prom = this.backend.getOutOfBandMembers(msg.args[0]);
+                break;
+            case 'clearOutOfBandMembers':
+                prom = this.backend.clearOutOfBandMembers(msg.args[0]);
+                break;
+            case 'setOutOfBandMembers':
+                prom = this.backend.setOutOfBandMembers(msg.args[0], msg.args[1]);
+                break;
+            case 'getClientOptions':
+                prom = this.backend.getClientOptions();
+                break;
+            case 'storeClientOptions':
+                prom = this.backend.storeClientOptions(msg.args[0]);
+                break;
        }

        if (prom === undefined) {
-            postMessage({
+            this.postMessage({
                command: 'cmd_fail',
                seq: msg.seq,
                // Can't be an Error because they're not structured cloneable
@@ -113,7 +135,10 @@ class IndexedDBStoreWorker {
                command: 'cmd_fail',
                seq: msg.seq,
                // Just send a string because Error objects aren't cloneable
-                error: "Error running command",
+                error: {
+                    message: err.message,
+                    name: err.name,
+                },
            });
        });
    }
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import q from "q";
+import Promise from 'bluebird';
 import {MatrixInMemoryStore} from "./memory";
 import utils from "../utils";
 import LocalIndexedDBStoreBackend from "./indexeddb-local-backend.js";
@@ -115,12 +116,16 @@ utils.inherits(IndexedDBStore, MatrixInMemoryStore);
  */
 IndexedDBStore.prototype.startup = function() {
    if (this.startedUp) {
-        return q();
+        console.log(`IndexedDBStore.startup: already started`);
+        return Promise.resolve();
    }

+    console.log(`IndexedDBStore.startup: connecting to backend`);
    return this.backend.connect().then(() => {
+        console.log(`IndexedDBStore.startup: loading presence events`);
        return this.backend.getUserPresenceEvents();
    }).then((userPresenceEvents) => {
+        console.log(`IndexedDBStore.startup: processing presence events`);
        userPresenceEvents.forEach(([userId, rawEvent]) => {
            const u = new User(userId);
            if (rawEvent) {
@@ -141,6 +146,19 @@ IndexedDBStore.prototype.getSavedSync = function() {
    return this.backend.getSavedSync();
 };

+/** @return {Promise<bool>} whether or not the database was newly created in this session. */
+IndexedDBStore.prototype.isNewlyCreated = function() {
+    return this.backend.isNewlyCreated();
+};
+
+/**
+ * @return {Promise} If there is a saved sync, the nextBatch token
+ * for this sync, otherwise null.
+ */
+IndexedDBStore.prototype.getSavedSyncToken = function() {
+    return this.backend.getNextBatchToken();
+},
+
 /**
 * Delete all data from this store.
 * @return {Promise} Resolves if the data was deleted from the database.
@@ -150,21 +168,35 @@ IndexedDBStore.prototype.deleteAllData = function() {
    return this.backend.clearDatabase().then(() => {
        console.log("Deleted indexeddb data.");
    }, (err) => {
-        console.error("Failed to delete indexeddb data: ", err);
+        console.error(`Failed to delete indexeddb data: ${err}`);
        throw err;
    });
 };

+/**
+ * Whether this store would like to save its data
+ * Note that obviously whether the store wants to save or
+ * not could change between calling this function and calling
+ * save().
+ *
+ * @return {boolean} True if calling save() will actually save
+ *     (at the time this function is called).
+ */
+IndexedDBStore.prototype.wantsSave = function() {
+    const now = Date.now();
+    return now - this._syncTs > WRITE_DELAY_MS;
+};
+
 /**
 * Possibly write data to the database.
- * @return {Promise} Promise resolves after the write completes.
+ * @return {Promise} Promise resolves after the write completes
+ *     (or immediately if no write is performed)
 */
 IndexedDBStore.prototype.save = function() {
-    const now = Date.now();
-    if (now - this._syncTs > WRITE_DELAY_MS) {
+    if (this.wantsSave()) {
        return this._reallySave();
    }
-    return q();
+    return Promise.resolve();
 };

 IndexedDBStore.prototype._reallySave = function() {
@@ -192,4 +224,39 @@ IndexedDBStore.prototype.setSyncData = function(syncData) {
    return this.backend.setSyncData(syncData);
 };

+/**
+ * Returns the out-of-band membership events for this room that
+ * were previously loaded.
+ * @param {string} roomId
+ * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+ * @returns {null} in case the members for this room haven't been stored yet
+ */
+IndexedDBStore.prototype.getOutOfBandMembers = function(roomId) {
+    return this.backend.getOutOfBandMembers(roomId);
+};
+
+/**
+ * Stores the out-of-band membership events for this room. Note that
+ * it still makes sense to store an empty array as the OOB status for the room is
+ * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+ * @param {string} roomId
+ * @param {event[]} membershipEvents the membership events to store
+ * @returns {Promise} when all members have been stored
+ */
+IndexedDBStore.prototype.setOutOfBandMembers = function(roomId, membershipEvents) {
+    return this.backend.setOutOfBandMembers(roomId, membershipEvents);
+};
+
+IndexedDBStore.prototype.clearOutOfBandMembers = function(roomId) {
+    return this.backend.clearOutOfBandMembers(roomId);
+};
+
+IndexedDBStore.prototype.getClientOptions = function() {
+    return this.backend.getClientOptions();
+};
+
+IndexedDBStore.prototype.storeClientOptions = function(options) {
+    return this.backend.storeClientOptions(options);
+};
+
 module.exports.IndexedDBStore = IndexedDBStore;
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,9 +20,9 @@ limitations under the License.
 * This is an internal module. See {@link MatrixInMemoryStore} for the public class.
 * @module store/memory
 */
- const utils = require("../utils");
- const User = require("../models/user");
- const q = require("q");
+const utils = require("../utils");
+const User = require("../models/user");
+import Promise from 'bluebird';

 /**
 * Construct a new in-memory data store for the Matrix Client.
@@ -35,6 +36,9 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
    this.rooms = {
        // roomId: Room
    };
+    this.groups = {
+        // groupId: Group
+    };
    this.users = {
        // userId: User
    };
@@ -48,6 +52,10 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
        // type : content
    };
    this.localStorage = opts.localStorage;
+    this._oobMembers = {
+        // roomId: [member events]
+    };
+    this._clientOptions = {};
 };

 module.exports.MatrixInMemoryStore.prototype = {
@@ -60,6 +68,10 @@ module.exports.MatrixInMemoryStore.prototype = {
        return this.syncToken;
    },

+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },

    /**
     * Set the token to stream from.
@@ -69,6 +81,31 @@ module.exports.MatrixInMemoryStore.prototype = {
        this.syncToken = token;
    },

+    /**
+     * Store the given room.
+     * @param {Group} group The group to be stored
+     */
+    storeGroup: function(group) {
+        this.groups[group.groupId] = group;
+    },
+
+    /**
+     * Retrieve a group by its group ID.
+     * @param {string} groupId The group ID.
+     * @return {Group} The group or null.
+     */
+    getGroup: function(groupId) {
+        return this.groups[groupId] || null;
+    },
+
+    /**
+     * Retrieve all known groups.
+     * @return {Group[]} A list of groups, which may be empty.
+     */
+    getGroups: function() {
+        return utils.values(this.groups);
+    },
+
    /**
     * Store the given room.
     * @param {Room} room The room to be stored. All properties must be stored.
@@ -284,7 +321,16 @@ module.exports.MatrixInMemoryStore.prototype = {
     * @return {Promise} An immediately resolved promise.
     */
    setSyncData: function(syncData) {
-        return q();
+        return Promise.resolve();
+    },
+
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
    },

    /**
@@ -297,7 +343,7 @@ module.exports.MatrixInMemoryStore.prototype = {
     * @return {Promise} An immediately resolved promise.
     */
    startup: function() {
-        return q();
+        return Promise.resolve();
    },

    /**
@@ -306,7 +352,15 @@ module.exports.MatrixInMemoryStore.prototype = {
     * is no saved sync data.
     */
    getSavedSync: function() {
-        return q(null);
+        return Promise.resolve(null);
+    },
+
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
    },

    /**
@@ -329,6 +383,37 @@ module.exports.MatrixInMemoryStore.prototype = {
        this.accountData = {
            // type : content
        };
-        return q();
+        return Promise.resolve();
+    },
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return Promise.resolve(this._oobMembers[roomId] || null);
+    },
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        this._oobMembers[roomId] = membershipEvents;
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve(this._clientOptions);
+    },
+
+    storeClientOptions: function(options) {
+        this._clientOptions = Object.assign({}, options);
+        return Promise.resolve();
    },
 };
@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -48,17 +50,18 @@ function WebStorageSessionStore(webStore) {
 }

 WebStorageSessionStore.prototype = {
-
    /**
-     * Store the end to end account for the logged-in user.
-     * @param {string} account Base64 encoded account.
+     * Remove the stored end to end account for the logged-in user.
     */
-    storeEndToEndAccount: function(account) {
-        this.store.setItem(KEY_END_TO_END_ACCOUNT, account);
+    removeEndToEndAccount: function() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
    },

    /**
     * Load the end to end account for the logged-in user.
+     * Note that the end-to-end account is now stored in the
+     * crypto store rather than here: this remains here so
+     * old sessions can be migrated out of the session store.
     * @return {?string} Base64 encoded account.
     */
    getEndToEndAccount: function() {
@@ -66,59 +69,24 @@ WebStorageSessionStore.prototype = {
    },

    /**
-     * Store a flag indicating that we have announced the new device.
+     * Retrieves the known devices for all users.
+     * @return {object} A map from user ID to map of device ID to keys for the device.
     */
-    setDeviceAnnounced: function() {
-        this.store.setItem(KEY_END_TO_END_ANNOUNCED, "true");
-    },
-
-    /**
-     * Check if the "device announced" flag is set
-     *
-     * @return {boolean} true if the "device announced" flag has been set.
-     */
-    getDeviceAnnounced: function() {
-        return this.store.getItem(KEY_END_TO_END_ANNOUNCED) == "true";
-    },
-
-    /**
-     * Stores the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @param {object} devices A map from device ID to keys for the device.
-     */
-    storeEndToEndDevicesForUser: function(userId, devices) {
-        setJsonItem(this.store, keyEndToEndDevicesForUser(userId), devices);
-    },
-
-    /**
-     * Retrieves the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @return {object} A map from device ID to keys for the device.
-     */
-    getEndToEndDevicesForUser: function(userId) {
-        return getJsonItem(this.store, keyEndToEndDevicesForUser(userId));
-    },
-
-    storeEndToEndDeviceTrackingStatus: function(statusMap) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS, statusMap);
+    getAllEndToEndDevices: function() {
+        const prefix = keyEndToEndDevicesForUser('');
+        const devices = {};
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            const userId = key.substr(prefix.length);
+            if (key.startsWith(prefix)) devices[userId] = getJsonItem(this.store, key);
+        }
+        return devices;
    },

    getEndToEndDeviceTrackingStatus: function() {
        return getJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
    },

-    /**
-     * Store the sync token corresponding to the device list.
-     *
-     * This is used when starting the client, to get a list of the users who
-     * have changed their device list since the list time we were running.
-     *
-     * @param {String?} token
-     */
-    storeEndToEndDeviceSyncToken: function(token) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN, token);
-    },
-
    /**
     * Get the sync token corresponding to the device list.
     *
@@ -129,17 +97,12 @@ WebStorageSessionStore.prototype = {
    },

    /**
-     * Store a session between the logged-in user and another device
-     * @param {string} deviceKey The public key of the other device.
-     * @param {string} sessionId The ID for this end-to-end session.
-     * @param {string} session Base64 encoded end-to-end session.
+     * Removes all end to end device data from the store
     */
-    storeEndToEndSession: function(deviceKey, sessionId, session) {
-        const sessions = this.getEndToEndSessions(deviceKey) || {};
-        sessions[sessionId] = session;
-        setJsonItem(
-            this.store, keyEndToEndSessions(deviceKey), sessions,
-        );
+    removeEndToEndDeviceData: function() {
+        removeByPrefix(this.store, keyEndToEndDevicesForUser(''));
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN);
    },

    /**
@@ -152,6 +115,29 @@ WebStorageSessionStore.prototype = {
        return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
    },

+    /**
+     * Retrieve all end-to-end sessions between the logged-in user and other
+     * devices.
+     * @return {object} A map of {deviceKey -> {sessionId -> session pickle}}
+     */
+    getAllEndToEndSessions: function() {
+        const deviceKeys = getKeysWithPrefix(this.store, keyEndToEndSessions(''));
+        const results = {};
+        for (const k of deviceKeys) {
+            const unprefixedKey = k.substr(keyEndToEndSessions('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
+    },
+
+    /**
+     * Remove all end-to-end sessions from the store
+     * This is used after migrating sessions awat from the sessions store.
+     */
+    removeAllEndToEndSessions: function() {
+        removeByPrefix(this.store, keyEndToEndSessions(''));
+    },
+
    /**
     * Retrieve a list of all known inbound group sessions
     *
@@ -183,34 +169,43 @@ WebStorageSessionStore.prototype = {
        return this.store.getItem(key);
    },

-    storeEndToEndInboundGroupSession: function(senderKey, sessionId, pickledSession) {
-        const key = keyEndToEndInboundGroupSession(senderKey, sessionId);
-        return this.store.setItem(key, pickledSession);
+    removeAllEndToEndInboundGroupSessions: function() {
+        removeByPrefix(this.store, E2E_PREFIX + 'inboundgroupsessions/');
    },

    /**
-     * Store the end-to-end state for a room.
-     * @param {string} roomId The room's ID.
-     * @param {object} roomInfo The end-to-end info for the room.
+     * Get the end-to-end state for all rooms
+     * @return {object} roomId -> object with the end-to-end info for the room.
     */
-    storeEndToEndRoom: function(roomId, roomInfo) {
-        setJsonItem(this.store, keyEndToEndRoom(roomId), roomInfo);
+    getAllEndToEndRooms: function() {
+        const roomKeys = getKeysWithPrefix(this.store, keyEndToEndRoom(''));
+        const results = {};
+        for (const k of roomKeys) {
+            const unprefixedKey = k.substr(keyEndToEndRoom('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
    },

-    /**
-     * Get the end-to-end state for a room
-     * @param {string} roomId The room's ID.
-     * @return {object} The end-to-end info for the room.
-     */
-    getEndToEndRoom: function(roomId) {
-        return getJsonItem(this.store, keyEndToEndRoom(roomId));
+    removeAllEndToEndRooms: function() {
+        removeByPrefix(this.store, keyEndToEndRoom(''));
+    },
+
+    setLocalTrustedBackupPubKey: function(pubkey) {
+        this.store.setItem(KEY_END_TO_END_TRUSTED_BACKUP_PUBKEY, pubkey);
+    },
+
+    // XXX: This store is deprecated really, but added this as a temporary
+    // thing until cross-signing lands.
+    getLocalTrustedBackupPubKey: function() {
+        return this.store.getItem(KEY_END_TO_END_TRUSTED_BACKUP_PUBKEY);
    },
 };

 const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
-const KEY_END_TO_END_ANNOUNCED = E2E_PREFIX + "announced";
 const KEY_END_TO_END_DEVICE_SYNC_TOKEN = E2E_PREFIX + "device_sync_token";
 const KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS = E2E_PREFIX + "device_tracking";
+const KEY_END_TO_END_TRUSTED_BACKUP_PUBKEY = E2E_PREFIX + "trusted_backup_pubkey";

 function keyEndToEndDevicesForUser(userId) {
    return E2E_PREFIX + "devices/" + userId;
@@ -240,8 +235,24 @@ function getJsonItem(store, key) {
    return null;
 }

-function setJsonItem(store, key, val) {
-    store.setItem(key, JSON.stringify(val));
+function getKeysWithPrefix(store, prefix) {
+    const results = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) results.push(key);
+    }
+    return results;
+}
+
+function removeByPrefix(store, prefix) {
+    const toRemove = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) toRemove.push(key);
+    }
+    for (const key of toRemove) {
+        store.removeItem(key);
+    }
 }

 function debuglog() {
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 "use strict";
-import q from "q";
+import Promise from 'bluebird';
 /**
 * This is an internal module.
 * @module store/stub
@@ -31,6 +32,11 @@ function StubStore() {

 StubStore.prototype = {

+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },
+
    /**
     * Get the sync token.
     * @return {string}
@@ -47,6 +53,30 @@ StubStore.prototype = {
        this.fromToken = token;
    },

+    /**
+     * No-op.
+     * @param {Group} group
+     */
+    storeGroup: function(group) {
+    },
+
+    /**
+     * No-op.
+     * @param {string} groupId
+     * @return {null}
+     */
+    getGroup: function(groupId) {
+        return null;
+    },
+
+    /**
+     * No-op.
+     * @return {Array} An empty array.
+     */
+    getGroups: function() {
+        return [];
+    },
+
    /**
     * No-op.
     * @param {Room} room
@@ -189,7 +219,16 @@ StubStore.prototype = {
     * @return {Promise} An immediately resolved promise.
     */
    setSyncData: function(syncData) {
-        return q();
+        return Promise.resolve();
+    },
+
+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
    },

    /**
@@ -202,7 +241,7 @@ StubStore.prototype = {
     * @return {Promise} An immediately resolved promise.
     */
    startup: function() {
-        return q();
+        return Promise.resolve();
    },

    /**
@@ -211,7 +250,15 @@ StubStore.prototype = {
     * is no saved sync data.
     */
    getSavedSync: function() {
-        return q(null);
+        return Promise.resolve(null);
+    },
+
+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
    },

    /**
@@ -220,7 +267,27 @@ StubStore.prototype = {
     * @return {Promise} An immediately resolved promise.
     */
    deleteAllData: function() {
-        return q();
+        return Promise.resolve();
+    },
+
+    getOutOfBandMembers: function() {
+        return Promise.resolve(null);
+    },
+
+    setOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    clearOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve();
+    },
+
+    storeClientOptions: function() {
+        return Promise.resolve();
    },
 };

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,7 +34,6 @@ import utils from "./utils";
 * rather than asking the server to do an initial sync on startup.
 */
 class SyncAccumulator {
-
    /**
     * @param {Object} opts
     * @param {Number=} opts.maxTimelineEntries The ideal maximum number of
@@ -62,6 +62,11 @@ class SyncAccumulator {
            //       { event: $event, token: null|token },
            //       ...
            //    ],
+            //    _summary: {
+            //       m.heroes: [ $user_id ],
+            //       m.joined_member_count: $count,
+            //       m.invited_member_count: $count
+            //    },
            //    _accountData: { $event_type: json },
            //    _unreadNotifications: { ... unread_notifications JSON ... },
            //    _readReceipts: { $user_id: { data: $json, eventId: $event_id }}
@@ -72,10 +77,18 @@ class SyncAccumulator {
        // coherent /sync response and know at what point they should be
        // streaming from without losing events.
        this.nextBatch = null;
+
+        // { ('invite'|'join'|'leave'): $groupId: { ... sync 'group' data } }
+        this.groups = {
+            invite: {},
+            join: {},
+            leave: {},
+        };
    }

    accumulate(syncResponse) {
        this._accumulateRooms(syncResponse);
+        this._accumulateGroups(syncResponse);
        this._accumulateAccountData(syncResponse);
        this.nextBatch = syncResponse.next_batch;
    }
@@ -233,6 +246,7 @@ class SyncAccumulator {
                _timeline: [],
                _accountData: Object.create(null),
                _unreadNotifications: {},
+                _summary: {},
                _readReceipts: {},
            };
        }
@@ -249,6 +263,17 @@ class SyncAccumulator {
        if (data.unread_notifications) {
            currentData._unreadNotifications = data.unread_notifications;
        }
+        if (data.summary) {
+            const HEROES_KEY = "m.heroes";
+            const INVITED_COUNT_KEY = "m.invited_member_count";
+            const JOINED_COUNT_KEY = "m.joined_member_count";
+
+            const acc = currentData._summary;
+            const sum = data.summary;
+            acc[HEROES_KEY] = sum[HEROES_KEY] || acc[HEROES_KEY];
+            acc[JOINED_COUNT_KEY] = sum[JOINED_COUNT_KEY] || acc[JOINED_COUNT_KEY];
+            acc[INVITED_COUNT_KEY] = sum[INVITED_COUNT_KEY] || acc[INVITED_COUNT_KEY];
+        }

        if (data.ephemeral && data.ephemeral.events) {
            data.ephemeral.events.forEach((e) => {
@@ -336,6 +361,44 @@ class SyncAccumulator {
        }
    }

+    /**
+     * Accumulate incremental /sync group data.
+     * @param {Object} syncResponse the complete /sync JSON
+     */
+    _accumulateGroups(syncResponse) {
+        if (!syncResponse.groups) {
+            return;
+        }
+        if (syncResponse.groups.invite) {
+            Object.keys(syncResponse.groups.invite).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "invite", syncResponse.groups.invite[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.join) {
+            Object.keys(syncResponse.groups.join).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "join", syncResponse.groups.join[groupId],
+                );
+            });
+        }
+        if (syncResponse.groups.leave) {
+            Object.keys(syncResponse.groups.leave).forEach((groupId) => {
+                this._accumulateGroup(
+                    groupId, "leave", syncResponse.groups.leave[groupId],
+                );
+            });
+        }
+    }
+
+    _accumulateGroup(groupId, category, data) {
+        for (const cat of ['invite', 'join', 'leave']) {
+            delete this.groups[cat][groupId];
+        }
+        this.groups[category][groupId] = data;
+    }
+
    /**
     * Return everything under the 'rooms' key from a /sync response which
     * represents all room data that should be stored. This should be paired
@@ -381,6 +444,7 @@ class SyncAccumulator {
                    prev_batch: null,
                },
                unread_notifications: roomData._unreadNotifications,
+                summary: roomData._summary,
            };
            // Add account data
            Object.keys(roomData._accountData).forEach((evType) => {
@@ -470,9 +534,14 @@ class SyncAccumulator {
        return {
            nextBatch: this.nextBatch,
            roomsData: data,
+            groupsData: this.groups,
            accountData: accData,
        };
    }
+
+    getNextBatchToken() {
+        return this.nextBatch;
+    }
 }

 function setState(eventMap, event) {
--- a/Show More
+++ b/Show More