v0.14.0

Prepare changelog for v0.14.0
v0.14.0-rc.1
2018-11-19 15:09:43 +00:00 · 2018-11-19 15:09:42 +00:00 · 2018-11-15 15:06:39 +00:00 · 2018-11-15 15:06:39 +00:00 · 2018-11-14 18:21:01 +01:00 · 2018-11-14 18:19:24 +01:00
68 changed files with 7416 additions and 1975 deletions
@@ -1,3 +1,554 @@
+Changes in [0.14.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.0) (2018-11-19)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.14.0-rc.1...v0.14.0)
+
+ * No changes since rc.1
+
+Changes in [0.14.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.14.0-rc.1) (2018-11-15)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.13.1...v0.14.0-rc.1)
+
+BREAKING CHANGE
+----------------
+ 
+ * js-sdk now uses Olm 3.0. Apps using Olm must update to 3.0 to
+   continue using Olm with the js-sdk. The js-sdk will call Olm's
+   init() method when the client is started.
+
+All Changes
+-----------
+
+ * Prevent messages from being sent if other messages have failed to send
+   [\#781](https://github.com/matrix-org/matrix-js-sdk/pull/781)
+ * A unit test for olm
+   [\#777](https://github.com/matrix-org/matrix-js-sdk/pull/777)
+ * Set access_token and user_id after login in with username and password.
+   [\#778](https://github.com/matrix-org/matrix-js-sdk/pull/778)
+ * Add function to get currently joined rooms.
+   [\#779](https://github.com/matrix-org/matrix-js-sdk/pull/779)
+ * Remove the request-only stuff we don't need anymore
+   [\#775](https://github.com/matrix-org/matrix-js-sdk/pull/775)
+ * Manually construct query strings for browser-request instances
+   [\#770](https://github.com/matrix-org/matrix-js-sdk/pull/770)
+ * Fix: correctly check for crypto being present
+   [\#769](https://github.com/matrix-org/matrix-js-sdk/pull/769)
+ * Update babel-eslint to 8.1.1
+   [\#768](https://github.com/matrix-org/matrix-js-sdk/pull/768)
+ * Support `request` in the browser and support supplying servers to try in
+   joinRoom()
+   [\#764](https://github.com/matrix-org/matrix-js-sdk/pull/764)
+ * loglevel should be a normal dependency
+   [\#767](https://github.com/matrix-org/matrix-js-sdk/pull/767)
+ * Stop devicelist when client is stopped
+   [\#766](https://github.com/matrix-org/matrix-js-sdk/pull/766)
+ * Update to WebAssembly-powered Olm
+   [\#743](https://github.com/matrix-org/matrix-js-sdk/pull/743)
+ * Logging lib. Fixes #332
+   [\#763](https://github.com/matrix-org/matrix-js-sdk/pull/763)
+ * Use new stop() method on matrix-mock-request
+   [\#765](https://github.com/matrix-org/matrix-js-sdk/pull/765)
+
+Changes in [0.13.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.13.1) (2018-11-14)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.13.0...v0.13.1)
+
+ * Add function to get currently joined rooms.
+   [\#779](https://github.com/matrix-org/matrix-js-sdk/pull/779)
+
+Changes in [0.13.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.13.0) (2018-11-15)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.1...v0.13.0)
+
+BREAKING CHANGE
+----------------
+ * `MatrixClient::login` now sets client `access_token` and `user_id` following successful login with username and password.
+
+Changes in [0.12.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.1) (2018-10-29)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.1-rc.1...v0.12.1)
+
+ * No changes since rc.1
+
+Changes in [0.12.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.1-rc.1) (2018-10-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.0...v0.12.1-rc.1)
+
+ * Add repository type to package.json to make it valid
+   [\#762](https://github.com/matrix-org/matrix-js-sdk/pull/762)
+ * Add getMediaConfig()
+   [\#761](https://github.com/matrix-org/matrix-js-sdk/pull/761)
+ * add new examples, to be expanded into a post
+   [\#739](https://github.com/matrix-org/matrix-js-sdk/pull/739)
+
+Changes in [0.12.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.0) (2018-10-16)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.12.0-rc.1...v0.12.0)
+
+ * No changes since rc.1
+
+Changes in [0.12.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.12.0-rc.1) (2018-10-11)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.1...v0.12.0-rc.1)
+
+BREAKING CHANGES
+----------------
+ * If js-sdk finds data in the store that is incompatible with the options currently being used,
+   it will emit sync state ERROR with an error of type InvalidStoreError. It will also stop trying
+   to sync in this situation: the app must stop the client and then either clear the store or
+   change the options (in this case, enable or disable lazy loading of members) and then start
+   the client again.
+
+All Changes
+-----------
+
+ * never replace /sync'ed memberships with OOB ones
+   [\#760](https://github.com/matrix-org/matrix-js-sdk/pull/760)
+ * Don't fail to start up if lazy load check fails
+   [\#759](https://github.com/matrix-org/matrix-js-sdk/pull/759)
+ * Make e2e work on Edge
+   [\#754](https://github.com/matrix-org/matrix-js-sdk/pull/754)
+ * throw error with same name and message over idb worker boundary
+   [\#758](https://github.com/matrix-org/matrix-js-sdk/pull/758)
+ * Default to a room version of 1 when there is no room create event
+   [\#755](https://github.com/matrix-org/matrix-js-sdk/pull/755)
+ * Silence bluebird warnings
+   [\#757](https://github.com/matrix-org/matrix-js-sdk/pull/757)
+ * allow non-ff merge from release branch into master
+   [\#750](https://github.com/matrix-org/matrix-js-sdk/pull/750)
+ * Reject with the actual error on indexeddb error
+   [\#751](https://github.com/matrix-org/matrix-js-sdk/pull/751)
+ * Update mocha to v5
+   [\#744](https://github.com/matrix-org/matrix-js-sdk/pull/744)
+ * disable lazy loading for guests as they cant create filters
+   [\#748](https://github.com/matrix-org/matrix-js-sdk/pull/748)
+ * Revert "Add getMediaLimits to client"
+   [\#745](https://github.com/matrix-org/matrix-js-sdk/pull/745)
+
+Changes in [0.11.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.1) (2018-10-01)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.1-rc.1...v0.11.1)
+
+ * No changes since rc.1
+
+Changes in [0.11.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.1-rc.1) (2018-09-27)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.0...v0.11.1-rc.1)
+
+ * make usage of hub compatible with latest version (2.5)
+   [\#747](https://github.com/matrix-org/matrix-js-sdk/pull/747)
+ * Detect when lazy loading has been toggled in client.startClient
+   [\#746](https://github.com/matrix-org/matrix-js-sdk/pull/746)
+ * Add getMediaLimits to client
+   [\#644](https://github.com/matrix-org/matrix-js-sdk/pull/644)
+ * Split npm start into an init and watch script
+   [\#742](https://github.com/matrix-org/matrix-js-sdk/pull/742)
+ * Revert "room name should only take canonical alias into account"
+   [\#738](https://github.com/matrix-org/matrix-js-sdk/pull/738)
+ * fix display name disambiguation with LL
+   [\#737](https://github.com/matrix-org/matrix-js-sdk/pull/737)
+ * Introduce Room.myMembership event
+   [\#735](https://github.com/matrix-org/matrix-js-sdk/pull/735)
+ * room name should only take canonical alias into account
+   [\#733](https://github.com/matrix-org/matrix-js-sdk/pull/733)
+ * state events from context response were not wrapped in a MatrixEvent
+   [\#732](https://github.com/matrix-org/matrix-js-sdk/pull/732)
+ * Reduce amount of promises created when inserting members
+   [\#724](https://github.com/matrix-org/matrix-js-sdk/pull/724)
+ * dont wait for LL members to be stored to resolve the members
+   [\#726](https://github.com/matrix-org/matrix-js-sdk/pull/726)
+ * RoomState.members emitted with wrong argument order for OOB members
+   [\#728](https://github.com/matrix-org/matrix-js-sdk/pull/728)
+
+Changes in [0.11.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.0) (2018-09-10)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.11.0-rc.1...v0.11.0)
+
+BREAKING CHANGES
+----------------
+ * v0.11.0-rc.1 introduced some breaking changes - see the respective release notes.
+
+No changes since rc.1
+
+Changes in [0.11.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.11.0-rc.1) (2018-09-07)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9...v0.11.0-rc.1)
+
+ * Support for lazy loading members. This should improve performance for
+   users who joined big rooms a lot. Pass to `lazyLoadMembers = true` option when calling `startClient`.
+
+BREAKING CHANGES
+----------------
+
+ * `MatrixClient::startClient` now returns a Promise. No method should be called on the client before that promise resolves. Before this method didn't return anything.
+ * A new `CATCHUP` sync state, emitted by `MatrixClient#"sync"` and returned by `MatrixClient::getSyncState()`, when doing initial sync after the `ERROR` state. See `MatrixClient` documentation for details.
+ * `RoomState::maySendEvent('m.room.message', userId)` & `RoomState::maySendMessage(userId)` do not check the membership of the user anymore, only the power level. To check if the syncing user is allowed to write in a room, use `Room::maySendMessage()` as `RoomState` is not always aware of the syncing user's membership anymore, in case lazy loading of members is enabled.
+
+All Changes
+-----------
+
+ * Only emit CATCHUP if recovering from conn error
+   [\#727](https://github.com/matrix-org/matrix-js-sdk/pull/727)
+ * Fix docstring for sync data.error
+   [\#725](https://github.com/matrix-org/matrix-js-sdk/pull/725)
+ * Re-apply "Don't rely on members to query if syncing user can post to room"
+   [\#723](https://github.com/matrix-org/matrix-js-sdk/pull/723)
+ * Revert "Don't rely on members to query if syncing user can post to room"
+   [\#721](https://github.com/matrix-org/matrix-js-sdk/pull/721)
+ * Don't rely on members to query if syncing user can post to room
+   [\#717](https://github.com/matrix-org/matrix-js-sdk/pull/717)
+ * Fixes for room.guessDMUserId
+   [\#719](https://github.com/matrix-org/matrix-js-sdk/pull/719)
+ * Fix filepanel also filtering main timeline with LL turned on.
+   [\#716](https://github.com/matrix-org/matrix-js-sdk/pull/716)
+ * Remove lazy loaded members when leaving room
+   [\#711](https://github.com/matrix-org/matrix-js-sdk/pull/711)
+ * Fix: show spinner again while recovering from connection error
+   [\#702](https://github.com/matrix-org/matrix-js-sdk/pull/702)
+ * Add method to query LL state in client
+   [\#714](https://github.com/matrix-org/matrix-js-sdk/pull/714)
+ * Fix: also load invited members when lazy loading members
+   [\#707](https://github.com/matrix-org/matrix-js-sdk/pull/707)
+ * Pass through function to discard megolm session
+   [\#704](https://github.com/matrix-org/matrix-js-sdk/pull/704)
+
+Changes in [0.10.9](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9) (2018-09-03)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9-rc.2...v0.10.9)
+
+ * No changes since rc.2
+
+Changes in [0.10.9-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9-rc.2) (2018-08-31)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.9-rc.1...v0.10.9-rc.2)
+
+ * Fix for "otherMember.getAvatarUrl is not a function"
+   [\#708](https://github.com/matrix-org/matrix-js-sdk/pull/708)
+
+Changes in [0.10.9-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.9-rc.1) (2018-08-30)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.8...v0.10.9-rc.1)
+
+ * Fix DM avatar
+   [\#706](https://github.com/matrix-org/matrix-js-sdk/pull/706)
+ * Lazy loading: avoid loading members at initial sync for e2e rooms
+   [\#699](https://github.com/matrix-org/matrix-js-sdk/pull/699)
+ * Improve setRoomEncryption guard against multiple m.room.encryption st…
+   [\#700](https://github.com/matrix-org/matrix-js-sdk/pull/700)
+ * Revert "Lazy loading: don't block on setting up room crypto"
+   [\#698](https://github.com/matrix-org/matrix-js-sdk/pull/698)
+ * Lazy loading: don't block on setting up room crypto
+   [\#696](https://github.com/matrix-org/matrix-js-sdk/pull/696)
+ * Add getVisibleRooms()
+   [\#695](https://github.com/matrix-org/matrix-js-sdk/pull/695)
+ * Add wrapper around getJoinedMemberCount()
+   [\#697](https://github.com/matrix-org/matrix-js-sdk/pull/697)
+ * Api to fetch events via /room/.../event/..
+   [\#694](https://github.com/matrix-org/matrix-js-sdk/pull/694)
+ * Support for room upgrades
+   [\#693](https://github.com/matrix-org/matrix-js-sdk/pull/693)
+ * Lazy loading of room members
+   [\#691](https://github.com/matrix-org/matrix-js-sdk/pull/691)
+
+Changes in [0.10.8](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.8) (2018-08-20)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.8-rc.1...v0.10.8)
+
+ * No changes since rc.1
+
+Changes in [0.10.8-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.8-rc.1) (2018-08-16)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.7...v0.10.8-rc.1)
+
+ * Add getVersion to Room
+   [\#689](https://github.com/matrix-org/matrix-js-sdk/pull/689)
+ * Add getSyncStateData()
+   [\#680](https://github.com/matrix-org/matrix-js-sdk/pull/680)
+ * Send sync error to listener
+   [\#679](https://github.com/matrix-org/matrix-js-sdk/pull/679)
+ * make sure room.tags is always a valid object to avoid crashes
+   [\#675](https://github.com/matrix-org/matrix-js-sdk/pull/675)
+ * Fix infinite spinner upon joining a room
+   [\#673](https://github.com/matrix-org/matrix-js-sdk/pull/673)
+
+Changes in [0.10.7](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.7) (2018-07-30)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.7-rc.1...v0.10.7)
+
+ * No changes since rc.1
+
+Changes in [0.10.7-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.7-rc.1) (2018-07-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.6...v0.10.7-rc.1)
+
+ * encrypt for invited users if history visibility allows.
+   [\#666](https://github.com/matrix-org/matrix-js-sdk/pull/666)
+
+Changes in [0.10.6](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.6) (2018-07-09)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.6-rc.1...v0.10.6)
+
+ * No changes since rc.1
+
+Changes in [0.10.6-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.6-rc.1) (2018-07-06)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.5...v0.10.6-rc.1)
+
+ * Expose event decryption error via Event.decrypted event
+   [\#665](https://github.com/matrix-org/matrix-js-sdk/pull/665)
+ * Add decryption error codes to base.DecryptionError
+   [\#663](https://github.com/matrix-org/matrix-js-sdk/pull/663)
+
+Changes in [0.10.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.5) (2018-06-29)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.5-rc.1...v0.10.5)
+
+ * No changes since rc.1
+
+Changes in [0.10.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.5-rc.1) (2018-06-21)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.4...v0.10.5-rc.1)
+
+ * fix auth header and filename=undefined
+   [\#659](https://github.com/matrix-org/matrix-js-sdk/pull/659)
+ * allow setting the output device for webrtc calls
+   [\#650](https://github.com/matrix-org/matrix-js-sdk/pull/650)
+ * arguments true and false are actually invalid
+   [\#596](https://github.com/matrix-org/matrix-js-sdk/pull/596)
+ * fix typo where `headers` was not being used and thus sent wrong content-type
+   [\#643](https://github.com/matrix-org/matrix-js-sdk/pull/643)
+ * fix some documentation typos
+   [\#642](https://github.com/matrix-org/matrix-js-sdk/pull/642)
+
+Changes in [0.10.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.4) (2018-06-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.4-rc.1...v0.10.4)
+
+ * No changes since rc.1
+
+Changes in [0.10.4-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.4-rc.1) (2018-06-06)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.3...v0.10.4-rc.1)
+
+ * check whether notif level is undefined, because `0` is falsey
+   [\#651](https://github.com/matrix-org/matrix-js-sdk/pull/651)
+
+Changes in [0.10.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.3) (2018-05-25)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.3-rc.1...v0.10.3)
+
+ * No changes since v0.10.3-rc.1
+
+Changes in [0.10.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.3-rc.1) (2018-05-24)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.2...v0.10.3-rc.1)
+
+BREAKING CHANGE
+---------------
+
+The deprecated 'callback' parameter has been removed from MatrixBaseApis.deactivateAccount
+
+ * Add `erase` option to deactivateAccount
+   [\#649](https://github.com/matrix-org/matrix-js-sdk/pull/649)
+ * Emit Session.no_consent when M_CONSENT_NOT_GIVEN received
+   [\#647](https://github.com/matrix-org/matrix-js-sdk/pull/647)
+
+Changes in [0.10.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.2) (2018-04-30)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.2-rc.1...v0.10.2)
+
+ * No changes from rc.1
+
+Changes in [0.10.2-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.2-rc.1) (2018-04-25)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.1...v0.10.2-rc.1)
+
+ * Ignore inserts of dup inbound group sessions, pt 2
+   [\#641](https://github.com/matrix-org/matrix-js-sdk/pull/641)
+ * Ignore inserts of duplicate inbound group sessions
+   [\#639](https://github.com/matrix-org/matrix-js-sdk/pull/639)
+ * Log IDB errors
+   [\#638](https://github.com/matrix-org/matrix-js-sdk/pull/638)
+ * Remove not very useful but veryv spammy log line
+   [\#632](https://github.com/matrix-org/matrix-js-sdk/pull/632)
+ * Switch event type to m.sticker.
+   [\#628](https://github.com/matrix-org/matrix-js-sdk/pull/628)
+
+Changes in [0.10.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.1) (2018-04-12)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0...v0.10.1)
+
+ * Log IDB errors
+   [\#638](https://github.com/matrix-org/matrix-js-sdk/pull/638)
+ * Ignore inserts of duplicate inbound group sessions
+   [\#639](https://github.com/matrix-org/matrix-js-sdk/pull/639)
+ * Ignore inserts of dup inbound group sessions, pt 2
+   [\#641](https://github.com/matrix-org/matrix-js-sdk/pull/641)
+
+Changes in [0.10.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0) (2018-04-11)
+==================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.2...v0.10.0)
+
+ * No changes
+
+Changes in [0.10.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.2) (2018-04-09)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.10.0-rc.1...v0.10.0-rc.2)
+
+ * Add wrapper for group join API
+ * Add wrapped API to set group join\_policy
+
+Changes in [0.10.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.10.0-rc.1) (2018-03-19)
+============================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.2...v0.10.0-rc.1)
+
+ * Fix duplicated state events in timeline from peek
+   [\#630](https://github.com/matrix-org/matrix-js-sdk/pull/630)
+ * Create indexeddb worker when starting the store
+   [\#627](https://github.com/matrix-org/matrix-js-sdk/pull/627)
+ * Fix indexeddb logging
+   [\#626](https://github.com/matrix-org/matrix-js-sdk/pull/626)
+ * Don't do /keys/changes on incremental sync
+   [\#625](https://github.com/matrix-org/matrix-js-sdk/pull/625)
+ * Don't mark devicelist dirty unnecessarily
+   [\#623](https://github.com/matrix-org/matrix-js-sdk/pull/623)
+ * Cache the joined member count for a room state
+   [\#619](https://github.com/matrix-org/matrix-js-sdk/pull/619)
+ * Fix JS doc
+   [\#618](https://github.com/matrix-org/matrix-js-sdk/pull/618)
+ * Precompute push actions for state events
+   [\#617](https://github.com/matrix-org/matrix-js-sdk/pull/617)
+ * Fix bug where global "Never send to unverified..." is ignored
+   [\#616](https://github.com/matrix-org/matrix-js-sdk/pull/616)
+ * Intern legacy top-level 'membership' field
+   [\#615](https://github.com/matrix-org/matrix-js-sdk/pull/615)
+ * Don't synthesize RR for m.room.redaction as causes the RR to go missing.
+   [\#598](https://github.com/matrix-org/matrix-js-sdk/pull/598)
+ * Make Events create Dates on demand
+   [\#613](https://github.com/matrix-org/matrix-js-sdk/pull/613)
+ * Stop cloning events when adding to state
+   [\#612](https://github.com/matrix-org/matrix-js-sdk/pull/612)
+ * De-dup code: use the initialiseState function
+   [\#611](https://github.com/matrix-org/matrix-js-sdk/pull/611)
+ * Create sentinel members on-demand
+   [\#610](https://github.com/matrix-org/matrix-js-sdk/pull/610)
+ * Some more doc on how sentinels work
+   [\#609](https://github.com/matrix-org/matrix-js-sdk/pull/609)
+ * Migrate room encryption store to crypto store
+   [\#597](https://github.com/matrix-org/matrix-js-sdk/pull/597)
+ * add parameter to getIdentityServerUrl to strip the protocol for invites
+   [\#600](https://github.com/matrix-org/matrix-js-sdk/pull/600)
+ * Move Device Tracking Data to Crypto Store
+   [\#594](https://github.com/matrix-org/matrix-js-sdk/pull/594)
+ * Optimise pushprocessor
+   [\#591](https://github.com/matrix-org/matrix-js-sdk/pull/591)
+ * Set event error before emitting
+   [\#592](https://github.com/matrix-org/matrix-js-sdk/pull/592)
+ * Add event type for stickers [WIP]
+   [\#590](https://github.com/matrix-org/matrix-js-sdk/pull/590)
+ * Migrate inbound sessions to cryptostore
+   [\#587](https://github.com/matrix-org/matrix-js-sdk/pull/587)
+ * Disambiguate names if they contain an mxid
+   [\#588](https://github.com/matrix-org/matrix-js-sdk/pull/588)
+ * Check for sessions in indexeddb before migrating
+   [\#585](https://github.com/matrix-org/matrix-js-sdk/pull/585)
+ * Emit an event for crypto store migration
+   [\#586](https://github.com/matrix-org/matrix-js-sdk/pull/586)
+ * Supporting fixes For making UnknownDeviceDialog not pop up automatically
+   [\#575](https://github.com/matrix-org/matrix-js-sdk/pull/575)
+ * Move sessions to the crypto store
+   [\#584](https://github.com/matrix-org/matrix-js-sdk/pull/584)
+ * Change crypto store transaction API
+   [\#582](https://github.com/matrix-org/matrix-js-sdk/pull/582)
+ * Add some missed copyright notices
+   [\#581](https://github.com/matrix-org/matrix-js-sdk/pull/581)
+ * Move Olm account to IndexedDB
+   [\#579](https://github.com/matrix-org/matrix-js-sdk/pull/579)
+ * Fix logging of DecryptionErrors to be more useful
+   [\#580](https://github.com/matrix-org/matrix-js-sdk/pull/580)
+ * [BREAKING] Change the behaviour of the unverfied devices blacklist flag
+   [\#568](https://github.com/matrix-org/matrix-js-sdk/pull/568)
+ * Support set_presence=offline for syncing
+   [\#557](https://github.com/matrix-org/matrix-js-sdk/pull/557)
+ * Consider cases where the sender may not redact their own event
+   [\#556](https://github.com/matrix-org/matrix-js-sdk/pull/556)
+
+Changes in [0.9.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.2) (2017-12-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.1...v0.9.2)
+
+
+Changes in [0.9.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.1) (2017-11-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0...v0.9.1)
+
+ * Fix the force TURN option
+   [\#577](https://github.com/matrix-org/matrix-js-sdk/pull/577)
+
+Changes in [0.9.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0) (2017-11-15)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.9.0-rc.1...v0.9.0)
+
+
+Changes in [0.9.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.9.0-rc.1) (2017-11-10)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5...v0.9.0-rc.1)
+
+ * Modify addRoomToGroup to allow setting isPublic, create alias
+   updateGroupRoomAssociation
+   [\#567](https://github.com/matrix-org/matrix-js-sdk/pull/567)
+ * Expose more functionality of pushprocessor
+   [\#565](https://github.com/matrix-org/matrix-js-sdk/pull/565)
+ * Function for working out notif trigger permission
+   [\#566](https://github.com/matrix-org/matrix-js-sdk/pull/566)
+ * keep track of event ID and timestamp of decrypted messages
+   [\#555](https://github.com/matrix-org/matrix-js-sdk/pull/555)
+ * Fix notifEvent computation
+   [\#564](https://github.com/matrix-org/matrix-js-sdk/pull/564)
+ * Fix power level of sentinel members
+   [\#563](https://github.com/matrix-org/matrix-js-sdk/pull/563)
+ * don't try to decrypt a redacted message (fixes vector-im/riot-web#3744)
+   [\#554](https://github.com/matrix-org/matrix-js-sdk/pull/554)
+ * Support room notifs
+   [\#562](https://github.com/matrix-org/matrix-js-sdk/pull/562)
+ * Fix the glob-to-regex code
+   [\#558](https://github.com/matrix-org/matrix-js-sdk/pull/558)
+
+Changes in [0.8.5](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5) (2017-10-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.5-rc.1...v0.8.5)
+
+ * Make unknown pushrule conditions not match
+   [\#559](https://github.com/matrix-org/matrix-js-sdk/pull/559)
+
+Changes in [0.8.5-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.5-rc.1) (2017-10-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.4...v0.8.5-rc.1)
+
+ * Implement wrapper API for removing a room from a group
+   [\#553](https://github.com/matrix-org/matrix-js-sdk/pull/553)
+ * Fix typo which resulted in stuck key download requests
+   [\#552](https://github.com/matrix-org/matrix-js-sdk/pull/552)
+ * Store group when it's created
+   [\#549](https://github.com/matrix-org/matrix-js-sdk/pull/549)
+ * Luke/groups remove rooms users from summary
+   [\#548](https://github.com/matrix-org/matrix-js-sdk/pull/548)
+ * Clean on prepublish
+   [\#546](https://github.com/matrix-org/matrix-js-sdk/pull/546)
+ * Implement wrapper APIs for adding rooms to group summary
+   [\#545](https://github.com/matrix-org/matrix-js-sdk/pull/545)
+
+Changes in [0.8.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.4) (2017-09-21)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3...v0.8.4)
+
+ * Fix build issue
+
+Changes in [0.8.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3) (2017-09-20)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.3-rc.1...v0.8.3)
+
+ * No changes
+
 Changes in [0.8.3-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.8.3-rc.1) (2017-09-19)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.8.2...v0.8.3-rc.1)
@@ -30,9 +30,61 @@ In Node.js
    console.log("Public Rooms: %s", JSON.stringify(data));
  });
 ```
+
 See below for how to include libolm to enable end-to-end-encryption. Please check
 [the Node.js terminal app](examples/node) for a more complex example.

+To start the client:
+
+```javascript
+await client.startClient({initialSyncLimit: 10});
+```
+
+You can perform a call to `/sync` to get the current state of the client:
+
+```javascript
+client.once('sync', function(state, prevState, res) {
+    if(state === 'PREPARED') {
+        console.log("prepared");
+    } else {
+        console.log(state);
+        process.exit(1);
+    }
+});
+```
+
+To send a message:
+
+```javascript
+var content = {
+    "body": "message text",
+    "msgtype": "m.text"
+};
+client.sendEvent("roomId", "m.room.message", content, "", (err, res) => {
+    console.log(err);
+});
+```
+
+To listen for message events:
+
+```javascript
+client.on("Room.timeline", function(event, room, toStartOfTimeline) {
+  if (event.getType() !== "m.room.message") {
+    return; // only use messages
+  }
+  console.log(event.event.content.body);
+});
+```
+
+By default, the `matrix-js-sdk` client uses the `MatrixInMemoryStore` to store events as they are received. For example to iterate through the currently stored timeline for a room:
+
+```javascript
+Object.keys(client.store.rooms).forEach((roomId) => {
+  client.getRoom(roomId).timeline.forEach(t => {
+      console.log(t.event);
+  });
+});
+```

 What does this SDK do?
 ----------------------
@@ -267,13 +319,13 @@ To provide the Olm library in a browser application:
 
 To provide the Olm library in a node.js application:

- * ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz``
+ * ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz``
   (replace the URL with the latest version you want to use from
    https://matrix.org/packages/npm/olm/)
 * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.

 If you want to package Olm as dependency for your node.js application, you
-can use ``npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz 
+can use ``npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz
 --save-optional`` (if your application also works without e2e crypto enabled)
 or ``--save`` (if it doesn't) to do so.

@@ -1,5 +1,17 @@
 var matrixcs = require("./lib/matrix");
-matrixcs.request(require("browser-request"));
+const request = require('browser-request');
+const queryString = require('qs');
+
+matrixcs.request(function(opts, fn) {
+    // We manually fix the query string for browser-request because
+    // it doesn't correctly handle cases like ?via=one&via=two. Instead
+    // we mimic `request`'s query string interface to make it all work
+    // as expected.
+    // browser-request will happily take the constructed string as the
+    // query string without trying to modify it further.
+    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
+    return request(opts, fn);
+});

 // just *accessing* indexedDB throws an exception in firefox with
 // indexeddb disabled.
@@ -202,9 +202,9 @@ function printRoomList() {
            dateStr = new Date(msg.getTs()).toISOString().replace(
                /T/, ' ').replace(/\..+/, '');
        }
-        var me = roomList[i].getMember(myUserId);
-        if (me) {
-            fmt = fmts[me.membership];
+        var myMembership = roomList[i].getMyMembership();
+        if (myMembership) {
+            fmt = fmts[myMembership];
        }
        var roomName = fixWidth(roomList[i].name, 25);
        print(
@@ -1,6 +1,6 @@
 {
  "name": "matrix-js-sdk",
-  "version": "0.8.3-rc.1",
+  "version": "0.14.0",
  "description": "Matrix Client-Server SDK for Javascript",
  "main": "index.js",
  "scripts": {
@@ -10,14 +10,18 @@
    "test": "npm run test:build && npm run test:run",
    "check": "npm run test:build && _mocha --recursive specbuild --colors",
    "gendoc": "babel --no-babelrc -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
-    "start": "babel -s -w -d lib src",
+    "start": "npm run start:init && npm run start:watch",
+    "start:watch": "babel -s -w --skip-initial-build -d lib src",
+    "start:init": "babel -s -d lib src",
+    "clean": "rimraf lib dist",
    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && uglifyjs -c -m -o dist/browser-matrix.min.js --source-map dist/browser-matrix.min.js.map --in-source-map dist/browser-matrix.js.map dist/browser-matrix.js",
    "dist": "npm run build",
    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 109 src spec",
-    "prepublish": "npm run build && git rev-parse HEAD > git-revision.txt"
+    "lint": "eslint --max-warnings 102 src spec",
+    "prepublish": "npm run clean && npm run build && git rev-parse HEAD > git-revision.txt"
  },
  "repository": {
+    "type": "git",
    "url": "https://github.com/matrix-org/matrix-js-sdk"
  },
  "keywords": [
@@ -49,18 +53,21 @@
  ],
  "dependencies": {
    "another-json": "^0.2.0",
+    "babel-runtime": "^6.26.0",
    "bluebird": "^3.5.0",
    "browser-request": "^0.3.3",
    "content-type": "^1.0.2",
-    "request": "^2.53.0"
+    "loglevel": "1.6.1",
+    "qs": "^6.5.2",
+    "request": "^2.88.0"
  },
  "devDependencies": {
    "babel-cli": "^6.18.0",
-    "babel-eslint": "^7.1.1",
+    "babel-eslint": "^8.1.1",
    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
    "babel-plugin-transform-runtime": "^6.23.0",
    "babel-preset-es2015": "^6.18.0",
-    "browserify": "^14.0.0",
+    "browserify": "^16.2.3",
    "browserify-shim": "^3.8.13",
    "eslint": "^3.13.1",
    "eslint-config-google": "^0.7.1",
@@ -69,14 +76,14 @@
    "istanbul": "^0.4.5",
    "jsdoc": "^3.5.5",
    "lolex": "^1.5.2",
-    "matrix-mock-request": "^1.2.0",
-    "mocha": "^3.2.0",
-    "mocha-jenkins-reporter": "^0.3.6",
+    "matrix-mock-request": "^1.2.2",
+    "mocha": "^5.2.0",
+    "mocha-jenkins-reporter": "^0.4.0",
    "rimraf": "^2.5.4",
    "source-map-support": "^0.4.11",
    "sourceify": "^0.1.0",
    "uglify-js": "^2.8.26",
-    "watchify": "^3.2.1"
+    "watchify": "^3.11.0"
  },
  "browserify": {
    "transform": [
@@ -245,7 +245,7 @@ release_text=`mktemp`
 echo "$tag" > "${release_text}"
 echo >> "${release_text}"
 cat "${latest_changes}" >> "${release_text}"
-hub release create $hubflags $assets -f "${release_text}" "$tag"
+hub release create $hubflags $assets -F "${release_text}" "$tag"

 if [ $dodist -eq 0 ]; then
    rm -rf "$builddir"
@@ -281,7 +281,7 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge --ff-only "$rel_branch"
+git merge "$rel_branch"

 # push master  and docs (if generated) to github
 git push origin master
@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -25,6 +26,7 @@ import testUtils from './test-utils';
 import MockHttpBackend from 'matrix-mock-request';
 import expect from 'expect';
 import Promise from 'bluebird';
+import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';

 /**
 * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -46,14 +48,19 @@ export default function TestClient(
    if (sessionStoreBackend === undefined) {
        sessionStoreBackend = new testUtils.MockStorageApi();
    }
-    this.storage = new sdk.WebStorageSessionStore(sessionStoreBackend);
+    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
+
+    // expose this so the tests can get to it
+    this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
+
    this.httpBackend = new MockHttpBackend();
    this.client = sdk.createClient({
        baseUrl: "http://" + userId + ".test.server",
        userId: userId,
        accessToken: accessToken,
        deviceId: deviceId,
-        sessionStore: this.storage,
+        sessionStore: sessionStore,
+        cryptoStore: this.cryptoStore,
        request: this.httpBackend.requestFn,
    });

@@ -95,9 +102,11 @@ TestClient.prototype.start = function() {

 /**
 * stop the client
+ * @return {Promise} Resolves once the mock http backend has finished all pending flushes
 */
 TestClient.prototype.stop = function() {
    this.client.stopClient();
+    return this.httpBackend.stop();
 };

 /**
@@ -1,3 +1,20 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import expect from 'expect';
 import Promise from 'bluebird';

@@ -80,7 +97,7 @@ describe("DeviceList management:", function() {
    });

    afterEach(function() {
-        aliceTestClient.stop();
+        return aliceTestClient.stop();
    });

    it("Alice shouldn't do a second /query for non-e2e-capable devices", function() {
@@ -151,9 +168,12 @@ describe("DeviceList management:", function() {
                   aliceTestClient.httpBackend.flush('/keys/query', 1).then(
                       () => aliceTestClient.httpBackend.flush('/send/', 1),
                   ),
+                   aliceTestClient.client._crypto._deviceList.saveIfDirty(),
               ]);
           }).then(() => {
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(1);
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   expect(data.syncToken).toEqual(1);
+               });

               // invalidate bob's and chris's device lists in separate syncs
               aliceTestClient.httpBackend.when('GET', '/sync').respond(200, {
@@ -185,19 +205,21 @@ describe("DeviceList management:", function() {
               return aliceTestClient.httpBackend.flush('/keys/query', 1);
           }).then((flushed) => {
               expect(flushed).toEqual(0);
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               if (bobStat != 1 && bobStat != 2) {
-                   throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
-
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   chrisStat);
-               }
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   if (bobStat != 1 && bobStat != 2) {
+                       throw new Error('Unexpected status for bob: wanted 1 or 2, got ' +
+                                       bobStat);
+                   }
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + chrisStat,
+                       );
+                   }
+               });

               // now add an expectation for a query for bob's devices, and let
               // it complete.
@@ -216,15 +238,18 @@ describe("DeviceList management:", function() {
               // wait for the client to stop processing the response
               return aliceTestClient.client.downloadKeys(['@bob:xyz']);
           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               expect(bobStat).toEqual(3);
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
-               if (chrisStat != 1 && chrisStat != 2) {
-                   throw new Error('Unexpected status for chris: wanted 1 or 2, got ' +
-                                   bobStat);
-               }
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   expect(bobStat).toEqual(3);
+                   const chrisStat = data.trackingStatus['@chris:abc'];
+                   if (chrisStat != 1 && chrisStat != 2) {
+                       throw new Error(
+                           'Unexpected status for chris: wanted 1 or 2, got ' + bobStat,
+                       );
+                   }
+               });

               // now let the query for chris's devices complete.
               return aliceTestClient.httpBackend.flush('/keys/query', 1);
@@ -234,16 +259,18 @@ describe("DeviceList management:", function() {
               // wait for the client to stop processing the response
               return aliceTestClient.client.downloadKeys(['@chris:abc']);
           }).then(() => {
-               const bobStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-               const chrisStat = aliceTestClient.storage
-                     .getEndToEndDeviceTrackingStatus()['@chris:abc'];
+               return aliceTestClient.client._crypto._deviceList.saveIfDirty();
+           }).then(() => {
+               aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                   const bobStat = data.trackingStatus['@bob:xyz'];
+                   const chrisStat = data.trackingStatus['@bob:xyz'];

-               expect(bobStat).toEqual(3);
-               expect(chrisStat).toEqual(3);
-               expect(aliceTestClient.storage.getEndToEndDeviceSyncToken()).toEqual(3);
+                   expect(bobStat).toEqual(3);
+                   expect(chrisStat).toEqual(3);
+                   expect(data.syncToken).toEqual(3);
+               });
           });
-       });
+       }).timeout(3000);

    // https://github.com/vector-im/riot-web/issues/4983
    describe("Alice should know she has stale device lists", () => {
@@ -262,13 +289,15 @@ describe("DeviceList management:", function() {
                },
            );
            await aliceTestClient.httpBackend.flush('/keys/query', 1);
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();

-            const bobStat = aliceTestClient.storage
-                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];

-            expect(bobStat).toBeGreaterThan(
-                0, "Alice should be tracking bob's device list",
-            );
+                expect(bobStat).toBeGreaterThan(
+                    0, "Alice should be tracking bob's device list",
+                );
+            });
        });

        it("when Bob leaves", async function() {
@@ -297,12 +326,15 @@ describe("DeviceList management:", function() {


            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();

-            const bobStat = aliceTestClient.storage
-                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-            expect(bobStat).toEqual(
-                0, "Alice should have marked bob's device list as untracked",
-            );
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
        });

        it("when Alice leaves", async function() {
@@ -330,12 +362,15 @@ describe("DeviceList management:", function() {
            );

            await aliceTestClient.flushSync();
+            await aliceTestClient.client._crypto._deviceList.saveIfDirty();

-            const bobStat = aliceTestClient.storage
-                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
-            expect(bobStat).toEqual(
-                0, "Alice should have marked bob's device list as untracked",
-            );
+            aliceTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                const bobStat = data.trackingStatus['@bob:xyz'];
+
+                expect(bobStat).toEqual(
+                    0, "Alice should have marked bob's device list as untracked",
+                );
+            });
        });

        it("when Bob leaves whilst Alice is offline", async function() {
@@ -344,23 +379,19 @@ describe("DeviceList management:", function() {
            const anotherTestClient = await createTestClient();

            try {
-                anotherTestClient.httpBackend.when('GET', '/keys/changes').respond(
-                    200, {
-                        changed: [],
-                        left: ['@bob:xyz'],
-                    },
-                );
                await anotherTestClient.start();
                anotherTestClient.httpBackend.when('GET', '/sync').respond(
                    200, getSyncResponse([]));
                await anotherTestClient.flushSync();
+                await anotherTestClient.client._crypto._deviceList.saveIfDirty();

-                const bobStat = anotherTestClient.storage
-                      .getEndToEndDeviceTrackingStatus()['@bob:xyz'];
+                anotherTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+                    const bobStat = data.trackingStatus['@bob:xyz'];

-                expect(bobStat).toEqual(
-                    0, "Alice should have marked bob's device list as untracked",
-                );
+                    expect(bobStat).toEqual(
+                        0, "Alice should have marked bob's device list as untracked",
+                    );
+                });
            } finally {
                anotherTestClient.stop();
            }
@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -154,11 +155,15 @@ function aliDownloadsKeys() {

    // check that the localStorage is updated as we expect (not sure this is
    // an integration test, but meh)
-    return Promise.all([p1, p2]).then(function() {
-        const devices = aliTestClient.storage.getEndToEndDevicesForUser(bobUserId);
-        expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
-        expect(devices[bobDeviceId].verified).
-            toBe(0); // DeviceVerification.UNVERIFIED
+    return Promise.all([p1, p2]).then(() => {
+        return aliTestClient.client._crypto._deviceList.saveIfDirty();
+    }).then(() => {
+        aliTestClient.cryptoStore.getEndToEndDeviceData(null, (data) => {
+            const devices = data.devices[bobUserId];
+            expect(devices[bobDeviceId].keys).toEqual(bobTestClient.deviceKeys.keys);
+            expect(devices[bobDeviceId].verified).
+                toBe(0); // DeviceVerification.UNVERIFIED
+        });
    });
 }

@@ -405,10 +410,10 @@ describe("MatrixClient crypto", function() {
    });

    afterEach(function() {
-        aliTestClient.stop();
        aliTestClient.httpBackend.verifyNoOutstandingExpectation();
-        bobTestClient.stop();
        bobTestClient.httpBackend.verifyNoOutstandingExpectation();
+
+        return Promise.all([aliTestClient.stop(), bobTestClient.stop()]);
    });

    it("Bob uploads device keys", function() {
@@ -30,6 +30,7 @@ describe("MatrixClient events", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("emissions", function() {
@@ -111,6 +111,7 @@ describe("getEventTimeline support", function() {
        if (client) {
            client.stopClient();
        }
+        return httpBackend.stop();
    });

    it("timeline support must be enabled to work", function(done) {
@@ -41,6 +41,7 @@ describe("MatrixClient", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    describe("uploadContent", function() {
@@ -159,7 +160,7 @@ describe("MatrixClient", function() {
    describe("joinRoom", function() {
        it("should no-op if you've already joined a room", function() {
            const roomId = "!foo:bar";
-            const room = new Room(roomId);
+            const room = new Room(roomId, userId);
            room.addLiveEvents([
                utils.mkMembership({
                    user: userId, room: roomId, mship: "join", event: true,
@@ -64,6 +64,7 @@ describe("MatrixClient opts", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    describe("without opts.store", function() {
@@ -94,7 +95,7 @@ describe("MatrixClient opts", function() {
            httpBackend.flush("/txn1", 1);
        });

-        it("should be able to sync / get new events", function(done) {
+        it("should be able to sync / get new events", async function() {
            const expectedEventTypes = [ // from /initialSync
                "m.room.message", "m.room.name", "m.room.member", "m.room.member",
                "m.room.create",
@@ -110,20 +111,16 @@ describe("MatrixClient opts", function() {
            httpBackend.when("GET", "/pushrules").respond(200, {});
            httpBackend.when("POST", "/filter").respond(200, { filter_id: "foo" });
            httpBackend.when("GET", "/sync").respond(200, syncData);
-            client.startClient();
-            httpBackend.flush("/pushrules", 1).then(function() {
-                return httpBackend.flush("/filter", 1);
-            }).then(function() {
-                return Promise.all([
-                    httpBackend.flush("/sync", 1),
-                    utils.syncPromise(client),
-                ]);
-            }).done(function() {
-                expect(expectedEventTypes.length).toEqual(
-                    0, "Expected to see event types: " + expectedEventTypes,
-                );
-                done();
-            });
+            await client.startClient();
+            await httpBackend.flush("/pushrules", 1);
+            await httpBackend.flush("/filter", 1);
+            await Promise.all([
+                httpBackend.flush("/sync", 1),
+                utils.syncPromise(client),
+            ]);
+            expect(expectedEventTypes.length).toEqual(
+                0, "Expected to see event types: " + expectedEventTypes,
+            );
        });
    });

@@ -36,6 +36,7 @@ describe("MatrixClient retrying", function() {

    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
+        return httpBackend.stop();
    });

    xit("should retry according to MatrixScheduler.retryFn", function() {
@@ -130,6 +130,7 @@ describe("MatrixClient room timelines", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("local echo events", function() {
@@ -38,6 +38,7 @@ describe("MatrixClient syncing", function() {
    afterEach(function() {
        httpBackend.verifyNoOutstandingExpectation();
        client.stopClient();
+        return httpBackend.stop();
    });

    describe("startClient", function() {
@@ -437,6 +438,34 @@ describe("MatrixClient syncing", function() {
            });
        });

+        // XXX: This test asserts that the js-sdk obeys the spec and treats state
+        // events that arrive in the incremental sync as if they preceeded the
+        // timeline events, however this breaks peeking, so it's disabled
+        // (see sync.js)
+        xit("should correctly interpret state in incremental sync.", function() {
+            httpBackend.when("GET", "/sync").respond(200, syncData);
+            httpBackend.when("GET", "/sync").respond(200, nextSyncData);
+
+            client.startClient();
+            return Promise.all([
+                httpBackend.flushAllExpected(),
+                awaitSyncEvent(2),
+            ]).then(function() {
+                const room = client.getRoom(roomOne);
+                const stateAtStart = room.getLiveTimeline().getState(
+                    EventTimeline.BACKWARDS,
+                );
+                const startRoomNameEvent = stateAtStart.getStateEvents('m.room.name', '');
+                expect(startRoomNameEvent.getContent().name).toEqual('Old room name');
+
+                const stateAtEnd = room.getLiveTimeline().getState(
+                    EventTimeline.FORWARDS,
+                );
+                const endRoomNameEvent = stateAtEnd.getStateEvents('m.room.name', '');
+                expect(endRoomNameEvent.getContent().name).toEqual('A new room name');
+            });
+        });
+
        xit("should update power levels for users in a room", function() {

        });
@@ -296,7 +296,7 @@ describe("megolm", function() {
    });

    afterEach(function() {
-        aliceTestClient.stop();
+        return aliceTestClient.stop();
    });

    it("Alice receives a megolm message", function() {
@@ -817,8 +817,14 @@ describe("megolm", function() {
                };
            });

+            // Grab the event that we'll need to resend
+            const room = aliceTestClient.client.getRoom(ROOM_ID);
+            const pendingEvents = room.getPendingEvents();
+            expect(pendingEvents.length).toEqual(1);
+            const unsentEvent = pendingEvents[0];
+
            return Promise.all([
-                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                aliceTestClient.client.resendEvent(unsentEvent, room),

                // the crypto stuff can take a while, so give the requests a whole second.
                aliceTestClient.httpBackend.flushAllExpected({
@@ -1,20 +1,23 @@

 "use strict";
 import 'source-map-support/register';
+import Crypto from '../../lib/crypto';
+import expect from 'expect';

 const sdk = require("../..");
-let Crypto;
-if (sdk.CRYPTO_ENABLED) {
-    Crypto = require("../../lib/crypto");
-}

-import expect from 'expect';
+const Olm = global.Olm;

 describe("Crypto", function() {
    if (!sdk.CRYPTO_ENABLED) {
        return;
    }
+
+    beforeEach(function(done) {
+        Olm.init().then(done);
+    });
+
    it("Crypto exposes the correct olm library version", function() {
-        expect(Crypto.getOlmVersion()[0]).toEqual(2);
+        expect(Crypto.getOlmVersion()[0]).toEqual(3);
    });
 });
@@ -1,6 +1,24 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import DeviceList from '../../../lib/crypto/DeviceList';
 import MockStorageApi from '../../MockStorageApi';
 import WebStorageSessionStore from '../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
 import testUtils from '../../test-utils';
 import utils from '../../../lib/utils';

@@ -40,13 +58,24 @@ const signedDeviceList = {
 describe('DeviceList', function() {
    let downloadSpy;
    let sessionStore;
+    let cryptoStore;
+    let deviceLists = [];

    beforeEach(function() {
        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this

+        deviceLists = [];
+
        downloadSpy = expect.createSpy();
        const mockStorage = new MockStorageApi();
        sessionStore = new WebStorageSessionStore(mockStorage);
+        cryptoStore = new MemoryCryptoStore();
+    });
+
+    afterEach(function() {
+        for (const dl of deviceLists) {
+            dl.stop();
+        }
    });

    function createTestDeviceList() {
@@ -56,7 +85,9 @@ describe('DeviceList', function() {
        const mockOlm = {
            verifySignature: function(key, message, signature) {},
        };
-        return new DeviceList(baseApis, sessionStore, mockOlm);
+        const dl = new DeviceList(baseApis, cryptoStore, sessionStore, mockOlm);
+        deviceLists.push(dl);
+        return dl;
    }

    it("should successfully download and store device keys", function() {
@@ -72,7 +103,7 @@ describe('DeviceList', function() {
        queryDefer1.resolve(utils.deepCopy(signedDeviceList));

        return prom1.then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
        });
    });
@@ -97,14 +128,15 @@ describe('DeviceList', function() {
        dl.invalidateUserDeviceList('@test1:sw1v.org');
        dl.refreshOutdatedDeviceLists();

-        // the first request completes
-        queryDefer1.resolve({
-            device_keys: {
-                '@test1:sw1v.org': {},
-            },
-        });
-
-        return prom1.then(() => {
+        dl.saveIfDirty().then(() => {
+            // the first request completes
+            queryDefer1.resolve({
+                device_keys: {
+                    '@test1:sw1v.org': {},
+                },
+            });
+            return prom1;
+        }).then(() => {
            // uh-oh; user restarts before second request completes. The new instance
            // should know we never got a complete device list.
            console.log("Creating new devicelist to simulate app reload");
@@ -121,7 +153,7 @@ describe('DeviceList', function() {
            // allow promise chain to complete
            return prom3;
        }).then(() => {
-            const storedKeys = sessionStore.getEndToEndDevicesForUser('@test1:sw1v.org');
+            const storedKeys = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
            expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
        });
    });
@@ -10,22 +10,19 @@ import Promise from 'bluebird';
 import sdk from '../../../..';
 import algorithms from '../../../../lib/crypto/algorithms';
 import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
 import MockStorageApi from '../../../MockStorageApi';
 import testUtils from '../../../test-utils';
-
-// Crypto and OlmDevice won't import unless we have global.Olm
-let OlmDevice;
-let Crypto;
-if (global.Olm) {
-    OlmDevice = require('../../../../lib/crypto/OlmDevice');
-    Crypto = require('../../../../lib/crypto');
-}
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+import Crypto from '../../../../lib/crypto';

 const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];

 const ROOM_ID = '!ROOM:ID';

+const Olm = global.Olm;
+
 describe("MegolmDecryption", function() {
    if (!global.Olm) {
        console.warn('Not running megolm unit tests: libolm not present');
@@ -45,8 +42,9 @@ describe("MegolmDecryption", function() {

        const mockStorage = new MockStorageApi();
        const sessionStore = new WebStorageSessionStore(mockStorage);
+        const cryptoStore = new MemoryCryptoStore(mockStorage);

-        const olmDevice = new OlmDevice(sessionStore);
+        const olmDevice = new OlmDevice(sessionStore, cryptoStore);

        megolmDecryption = new MegolmDecryption({
            userId: '@user:id',
@@ -67,7 +65,8 @@ describe("MegolmDecryption", function() {

    describe('receives some keys:', function() {
        let groupSession;
-        beforeEach(function() {
+        beforeEach(async function() {
+            await Olm.init();
            groupSession = new global.Olm.OutboundGroupSession();
            groupSession.create();

@@ -96,7 +95,7 @@ describe("MegolmDecryption", function() {
                },
            };

-            return event.attemptDecryption(mockCrypto).then(() => {
+            await event.attemptDecryption(mockCrypto).then(() => {
                megolmDecryption.onRoomKeyEvent(event);
            });
        });
@@ -181,5 +180,88 @@ describe("MegolmDecryption", function() {
                expect(payload.content.session_key).toExist();
            });
        });
+
+        it("can detect replay attacks", function() {
+            // trying to decrypt two different messages (marked by different
+            // event IDs or timestamps) using the same (sender key, session id,
+            // message index) triple should result in an exception being thrown
+            // as it should be detected as a replay attack.
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event1 = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            const successHandler = expect.createSpy();
+            const failureHandler = expect.createSpy()
+                .andCall((err) => {
+                    expect(err.toString()).toMatch(
+                        /Duplicate message index, possible replay attack/,
+                    );
+                });
+
+            return megolmDecryption.decryptEvent(event1).then((res) => {
+                const event2 = new MatrixEvent({
+                    type: 'm.room.encrypted',
+                    room_id: ROOM_ID,
+                    content: {
+                        algorithm: 'm.megolm.v1.aes-sha2',
+                        sender_key: "SENDER_CURVE25519",
+                        session_id: sessionId,
+                        ciphertext: cipherText,
+                    },
+                    event_id: "$event2",
+                    origin_server_ts: 1507754149000,
+                });
+
+                return megolmDecryption.decryptEvent(event2);
+            }).then(
+                successHandler,
+                failureHandler,
+            ).then(() => {
+                expect(successHandler).toNotHaveBeenCalled();
+                expect(failureHandler).toHaveBeenCalled();
+            });
+        });
+
+        it("allows re-decryption of the same event", function() {
+            // in contrast with the previous test, if the event ID and
+            // timestamp are the same, then it should not be considered a
+            // replay attack
+            const sessionId = groupSession.session_id();
+            const cipherText = groupSession.encrypt(JSON.stringify({
+                room_id: ROOM_ID,
+                content: 'testytest',
+            }));
+            const event = new MatrixEvent({
+                type: 'm.room.encrypted',
+                room_id: ROOM_ID,
+                content: {
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    sender_key: "SENDER_CURVE25519",
+                    session_id: sessionId,
+                    ciphertext: cipherText,
+                },
+                event_id: "$event1",
+                origin_server_ts: 1507753886000,
+            });
+
+            return megolmDecryption.decryptEvent(event).then((res) => {
+                return megolmDecryption.decryptEvent(event);
+                // test is successful if no exception is thrown
+            });
+        });
    });
 });
@@ -0,0 +1,90 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+try {
+    global.Olm = require('olm');
+} catch (e) {
+    console.warn("unable to run megolm tests: libolm not available");
+}
+
+import expect from 'expect';
+import WebStorageSessionStore from '../../../../lib/store/session/webstorage';
+import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
+import MockStorageApi from '../../../MockStorageApi';
+import testUtils from '../../../test-utils';
+
+import OlmDevice from '../../../../lib/crypto/OlmDevice';
+
+function makeOlmDevice() {
+    const mockStorage = new MockStorageApi();
+    const sessionStore = new WebStorageSessionStore(mockStorage);
+    const cryptoStore = new MemoryCryptoStore(mockStorage);
+    const olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    return olmDevice;
+}
+
+async function setupSession(initiator, opponent) {
+    await opponent.generateOneTimeKeys(1);
+    const keys = await opponent.getOneTimeKeys();
+    const firstKey = Object.values(keys['curve25519'])[0];
+
+    const sid = await initiator.createOutboundSession(
+        opponent.deviceCurve25519Key, firstKey,
+    );
+    return sid;
+}
+
+describe("OlmDecryption", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm unit tests: libolm not present');
+        return;
+    }
+
+    let aliceOlmDevice;
+    let bobOlmDevice;
+
+    beforeEach(async function() {
+        testUtils.beforeEach(this); // eslint-disable-line no-invalid-this
+
+        await global.Olm.init();
+
+        aliceOlmDevice = makeOlmDevice();
+        bobOlmDevice = makeOlmDevice();
+        await aliceOlmDevice.init();
+        await bobOlmDevice.init();
+    });
+
+    describe('olm', function() {
+        it("can decrypt messages", async function() {
+            const sid = await setupSession(aliceOlmDevice, bobOlmDevice);
+
+            const ciphertext = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sid,
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+
+            const result = await bobOlmDevice.createInboundSession(
+                aliceOlmDevice.deviceCurve25519Key,
+                ciphertext.type,
+                ciphertext.body,
+            );
+            expect(result.payload).toEqual(
+                "The olm or proteus is an aquatic salamander in the family Proteidae",
+            );
+        });
+    });
+});
@@ -132,12 +132,16 @@ describe("MatrixClient", function() {
        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
        store = [
            "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
-            "save", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
+            "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
            "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
            "getSyncAccumulator", "startup", "deleteAllData",
        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+        store.getClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.storeClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
+        store.isNewlyCreated = expect.createSpy().andReturn(Promise.resolve(true));
        client = new MatrixClient({
            baseUrl: "https://my.home.server",
            idBaseUrl: identityServerUrl,
@@ -181,7 +185,7 @@ describe("MatrixClient", function() {
        });
    });

-    it("should not POST /filter if a matching filter already exists", function(done) {
+    it("should not POST /filter if a matching filter already exists", async function() {
        httpLookups = [];
        httpLookups.push(PUSH_RULES_RESPONSE);
        httpLookups.push(SYNC_RESPONSE);
@@ -190,15 +194,19 @@ describe("MatrixClient", function() {
        const filter = new sdk.Filter(0, filterId);
        filter.setDefinition({"room": {"timeline": {"limit": 8}}});
        store.getFilter.andReturn(filter);
-        client.startClient();
-
-        client.on("sync", function syncListener(state) {
-            if (state === "SYNCING") {
-                expect(httpLookups.length).toEqual(0);
-                client.removeListener("sync", syncListener);
-                done();
-            }
+        const syncPromise = new Promise((resolve, reject) => {
+            client.on("sync", function syncListener(state) {
+                if (state === "SYNCING") {
+                    expect(httpLookups.length).toEqual(0);
+                    client.removeListener("sync", syncListener);
+                    resolve();
+                } else if (state === "ERROR") {
+                    reject(new Error("sync error"));
+                }
+            });
        });
+        await client.startClient();
+        await syncPromise;
    });

    describe("getSyncState", function() {
@@ -206,15 +214,18 @@ describe("MatrixClient", function() {
            expect(client.getSyncState()).toBe(null);
        });

-        it("should return the same sync state as emitted sync events", function(done) {
-            client.on("sync", function syncListener(state) {
-                expect(state).toEqual(client.getSyncState());
-                if (state === "SYNCING") {
-                    client.removeListener("sync", syncListener);
-                    done();
-                }
+        it("should return the same sync state as emitted sync events", async function() {
+            const syncingPromise = new Promise((resolve) => {
+                client.on("sync", function syncListener(state) {
+                    expect(state).toEqual(client.getSyncState());
+                    if (state === "SYNCING") {
+                        client.removeListener("sync", syncListener);
+                        resolve();
+                    }
+                });
            });
-            client.startClient();
+            await client.startClient();
+            await syncingPromise;
        });
    });

@@ -257,8 +268,8 @@ describe("MatrixClient", function() {
    });

    describe("retryImmediately", function() {
-        it("should return false if there is no request waiting", function() {
-            client.startClient();
+        it("should return false if there is no request waiting", async function() {
+            await client.startClient();
            expect(client.retryImmediately()).toBe(false);
        });

@@ -379,7 +390,7 @@ describe("MatrixClient", function() {
            client.startClient();
        });

-        it("should transition ERROR -> PREPARED after /sync if prev failed",
+        it("should transition ERROR -> CATCHUP after /sync if prev failed",
        function(done) {
            const expectedStates = [];
            acceptKeepalives = false;
@@ -402,7 +413,7 @@ describe("MatrixClient", function() {

            expectedStates.push(["RECONNECTING", null]);
            expectedStates.push(["ERROR", "RECONNECTING"]);
-            expectedStates.push(["PREPARED", "ERROR"]);
+            expectedStates.push(["CATCHUP", "ERROR"]);
            client.on("sync", syncChecker(expectedStates, done));
            client.startClient();
        });
@@ -24,6 +24,9 @@ describe('NotificationService', function() {
                            name: testDisplayName,
                        };
                    },
+                    getJoinedMemberCount: function() {
+                        return 0;
+                    },
                    members: {},
                },
            };
@@ -192,6 +192,15 @@ describe("RoomMember", function() {
        });
    });

+    describe("isOutOfBand", function() {
+        it("should be set by markOutOfBand", function() {
+            const member = new RoomMember();
+            expect(member.isOutOfBand()).toEqual(false);
+            member.markOutOfBand();
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+    });
+
    describe("setMembershipEvent", function() {
        const joinEvent = utils.mkMembership({
            event: true,
@@ -11,6 +11,9 @@ describe("RoomState", function() {
    const roomId = "!foo:bar";
    const userA = "@alice:bar";
    const userB = "@bob:bar";
+    const userC = "@cleo:bar";
+    const userLazy = "@lazy:bar";
+
    let state;

    beforeEach(function() {
@@ -78,8 +81,8 @@ describe("RoomState", function() {
    });

    describe("getSentinelMember", function() {
-        it("should return null if there is no member", function() {
-            expect(state.getSentinelMember("@no-one:here")).toEqual(null);
+        it("should return a member with the user id as name", function() {
+            expect(state.getSentinelMember("@no-one:here").name).toEqual("@no-one:here");
        });

        it("should return a member which doesn't change when the state is updated",
@@ -162,6 +165,7 @@ describe("RoomState", function() {
            ];
            let emitCount = 0;
            state.on("RoomState.newMember", function(ev, st, mem) {
+                expect(state.getMember(mem.userId)).toEqual(mem);
                expect(mem.userId).toEqual(memberEvents[emitCount].getSender());
                expect(mem.membership).toBeFalsy();  // not defined yet
                emitCount += 1;
@@ -222,7 +226,6 @@ describe("RoomState", function() {

        it("should call setPowerLevelEvent on a new RoomMember if power levels exist",
        function() {
-            const userC = "@cleo:bar";
            const memberEvent = utils.mkMembership({
                mship: "join", user: userC, room: roomId, event: true,
            });
@@ -262,6 +265,114 @@ describe("RoomState", function() {
        });
    });

+    describe("setOutOfBandMembers", function() {
+        it("should add a new member", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const member = state.getMember(userLazy);
+            expect(member.userId).toEqual(userLazy);
+            expect(member.isOutOfBand()).toEqual(true);
+        });
+
+        it("should have no effect when not in correct status", function() {
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should emit newMember when adding a member", function() {
+            const userLazy = "@oob:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.newMember', (_, __, member) => {
+                expect(member.userId).toEqual(userLazy);
+                eventReceived = true;
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+
+        it("should never overwrite existing members", function() {
+            const oobMemberEvent = utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            });
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            const memberA = state.getMember(userA);
+            expect(memberA.events.member.getId()).toNotEqual(oobMemberEvent.getId());
+            expect(memberA.isOutOfBand()).toEqual(false);
+        });
+
+        it("should emit members when updating a member", function() {
+            const doesntExistYetUserId = "@doesntexistyet:hs";
+            const oobMemberEvent = utils.mkMembership({
+                user: doesntExistYetUserId, mship: "join", room: roomId, event: true,
+            });
+            let eventReceived = false;
+            state.once('RoomState.members', (_, __, member) => {
+                expect(member.userId).toEqual(doesntExistYetUserId);
+                eventReceived = true;
+            });
+
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([oobMemberEvent]);
+            expect(eventReceived).toEqual(true);
+        });
+    });
+
+    describe("clone", function() {
+        it("should contain same information as original", function() {
+            // include OOB members in copy
+            state.markOutOfBandMembersStarted();
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            const copy = state.clone();
+            // check individual members
+            [userA, userB, userLazy].forEach((userId) => {
+                const member = state.getMember(userId);
+                const memberCopy = copy.getMember(userId);
+                expect(member.name).toEqual(memberCopy.name);
+                expect(member.isOutOfBand()).toEqual(memberCopy.isOutOfBand());
+            });
+            // check member keys
+            expect(Object.keys(state.members)).toEqual(Object.keys(copy.members));
+            // check join count
+            expect(state.getJoinedMemberCount()).toEqual(copy.getJoinedMemberCount());
+        });
+
+        it("should mark old copy as not waiting for out of band anymore", function() {
+            state.markOutOfBandMembersStarted();
+            const copy = state.clone();
+            copy.setOutOfBandMembers([utils.mkMembership({
+                user: userA, mship: "join", room: roomId, event: true,
+            })]);
+            // should have no effect as it should be marked in status finished just like copy
+            state.setOutOfBandMembers([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+            expect(state.getMember(userLazy)).toBeFalsy();
+        });
+
+        it("should return copy independent of original", function() {
+            const copy = state.clone();
+            copy.setStateEvents([utils.mkMembership({
+                user: userLazy, mship: "join", room: roomId, event: true,
+            })]);
+
+            expect(state.getMember(userLazy)).toBeFalsy();
+            expect(state.getJoinedMemberCount()).toEqual(2);
+            expect(copy.getJoinedMemberCount()).toEqual(3);
+        });
+    });
+
    describe("setTypingEvent", function() {
        it("should call setTypingEvent on each RoomMember", function() {
            const typingEvent = utils.mkEvent({
@@ -284,13 +395,6 @@ describe("RoomState", function() {
    });

    describe("maySendStateEvent", function() {
-        it("should say non-joined members may not send state",
-        function() {
-            expect(state.maySendStateEvent(
-                'm.room.name', "@nobody:nowhere",
-            )).toEqual(false);
-        });
-
        it("should say any member may send state with no power level event",
        function() {
            expect(state.maySendStateEvent('m.room.name', userA)).toEqual(true);
@@ -366,15 +470,117 @@ describe("RoomState", function() {
        });
    });

-    describe("maySendEvent", function() {
-        it("should say non-joined members may not send events",
-        function() {
-            expect(state.maySendEvent(
-                'm.room.message', "@nobody:nowhere",
-            )).toEqual(false);
-            expect(state.maySendMessage("@nobody:nowhere")).toEqual(false);
+    describe("getJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
        });

+        it("should update after adding joined member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("getInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should update after adding invited member", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(2);
+        });
+    });
+
+    describe("setJoinedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(1);
+            state.setJoinedMemberCount(100);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userA, room: roomId}),
+            ]);
+            state.setJoinedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "join",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getJoinedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("setInvitedMemberCount", function() {
+        beforeEach(() => {
+            state = new RoomState(roomId);
+        });
+
+        it("should, once used, override counting members from state", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(1);
+            state.setInvitedMemberCount(100);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+
+        it("should, once used, override counting members from state, " +
+        "also after clone", function() {
+            state.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userB, room: roomId}),
+            ]);
+            state.setInvitedMemberCount(100);
+            const copy = state.clone();
+            copy.setStateEvents([
+                utils.mkMembership({event: true, mship: "invite",
+                    user: userC, room: roomId}),
+            ]);
+            expect(state.getInvitedMemberCount()).toEqual(100);
+        });
+    });
+
+    describe("maySendEvent", function() {
        it("should say any member may send events with no power level event",
        function() {
            expect(state.maySendEvent('m.room.message', userA)).toEqual(true);
@@ -67,13 +67,14 @@ describe("Room", function() {

    describe("getMember", function() {
        beforeEach(function() {
-            // clobber members property with test data
-            room.currentState.members = {
-                "@alice:bar": {
-                    userId: userA,
-                    roomId: roomId,
-                },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": {
+                        userId: userA,
+                        roomId: roomId,
+                    },
+                }[userId];
+            });
        });

        it("should return null if the member isn't in current state", function() {
@@ -383,22 +384,25 @@ describe("Room", function() {
    });

    const resetTimelineTests = function(timelineSupport) {
-        const events = [
-            utils.mkMessage({
-                room: roomId, user: userA, msg: "A message", event: true,
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "New Room Name" },
-            }),
-            utils.mkEvent({
-                type: "m.room.name", room: roomId, user: userA, event: true,
-                content: { name: "Another New Name" },
-            }),
-        ];
+        let events = null;

        beforeEach(function() {
-            room = new Room(roomId, {timelineSupport: timelineSupport});
+            room = new Room(roomId, null, null, {timelineSupport: timelineSupport});
+            // set events each time to avoid resusing Event objects (which
+            // doesn't work because they get frozen)
+            events = [
+                utils.mkMessage({
+                    room: roomId, user: userA, msg: "A message", event: true,
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "New Room Name" },
+                }),
+                utils.mkEvent({
+                    type: "m.room.name", room: roomId, user: userA, event: true,
+                    content: { name: "Another New Name" },
+                }),
+            ];
        });

        it("should copy state from previous timeline", function() {
@@ -465,7 +469,7 @@ describe("Room", function() {

    describe("compareEventOrdering", function() {
        beforeEach(function() {
-            room = new Room(roomId, {timelineSupport: true});
+            room = new Room(roomId, null, null, {timelineSupport: true});
        });

        const events = [
@@ -567,72 +571,75 @@ describe("Room", function() {
    describe("hasMembershipState", function() {
        it("should return true for a matching userId and membership",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-                "@bob:bar": { userId: "@bob:bar", membership: "invite" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                    "@bob:bar": { userId: "@bob:bar", membership: "invite" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "invite")).toBe(true);
        });

        it("should return false if match membership but no match userId",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "join")).toBe(false);
        });

        it("should return false if match userId but no match membership",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@alice:bar", "ban")).toBe(false);
        });

        it("should return false if no match membership or userId",
        function() {
-            room.currentState.members = {
-                "@alice:bar": { userId: "@alice:bar", membership: "join" },
-            };
+            room.currentState.getMember.andCall(function(userId) {
+                return {
+                    "@alice:bar": { userId: "@alice:bar", membership: "join" },
+                }[userId];
+            });
            expect(room.hasMembershipState("@bob:bar", "invite")).toBe(false);
        });

        it("should return false if no members exist",
        function() {
-            room.currentState.members = {};
            expect(room.hasMembershipState("@foo:bar", "join")).toBe(false);
        });
    });

    describe("recalculate", function() {
-        let stateLookup = {
-            // event.type + "$" event.state_key : MatrixEvent
-        };
-
        const setJoinRule = function(rule) {
-            stateLookup["m.room.join_rules$"] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.join_rules", room: roomId, user: userA, content: {
                    join_rule: rule,
                }, event: true,
-            });
+            })]);
        };
        const setAliases = function(aliases, stateKey) {
            if (!stateKey) {
                stateKey = "flibble";
            }
-            stateLookup["m.room.aliases$" + stateKey] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
                    aliases: aliases,
                }, event: true,
-            });
+            })]);
        };
        const setRoomName = function(name) {
-            stateLookup["m.room.name$"] = utils.mkEvent({
+            room.addLiveEvents([utils.mkEvent({
                type: "m.room.name", room: roomId, user: userA, content: {
                    name: name,
                }, event: true,
-            });
+            })]);
        };
        const addMember = function(userId, state, opts) {
            if (!state) {
@@ -644,56 +651,14 @@ describe("Room", function() {
            opts.user = opts.user || userId;
            opts.skey = userId;
            opts.event = true;
-            stateLookup["m.room.member$" + userId] = utils.mkMembership(opts);
+            const event = utils.mkMembership(opts);
+            room.addLiveEvents([event]);
+            return event;
        };

        beforeEach(function() {
-            stateLookup = {};
-            room.currentState.getStateEvents.andCall(function(type, key) {
-                if (key === undefined) {
-                    const prefix = type + "$";
-                    const list = [];
-                    for (const stateBlob in stateLookup) {
-                        if (!stateLookup.hasOwnProperty(stateBlob)) {
-                            continue;
-                        }
-                        if (stateBlob.indexOf(prefix) === 0) {
-                            list.push(stateLookup[stateBlob]);
-                        }
-                    }
-                    return list;
-                } else {
-                    return stateLookup[type + "$" + key];
-                }
-            });
-            room.currentState.getMembers.andCall(function() {
-                const memberEvents = room.currentState.getStateEvents("m.room.member");
-                const members = [];
-                for (let i = 0; i < memberEvents.length; i++) {
-                    members.push({
-                        name: memberEvents[i].event.content &&
-                                memberEvents[i].event.content.displayname ?
-                                memberEvents[i].event.content.displayname :
-                                memberEvents[i].getStateKey(),
-                        userId: memberEvents[i].getStateKey(),
-                        events: { member: memberEvents[i] },
-                    });
-                }
-                return members;
-            });
-            room.currentState.getMember.andCall(function(userId) {
-                const memberEvent = room.currentState.getStateEvents(
-                    "m.room.member", userId,
-                );
-                return {
-                    name: memberEvent.event.content &&
-                            memberEvent.event.content.displayname ?
-                            memberEvent.event.content.displayname :
-                            memberEvent.getStateKey(),
-                    userId: memberEvent.getStateKey(),
-                    events: { member: memberEvent },
-                };
-            });
+            // no mocking
+            room = new Room(roomId, null, userA);
        });

        describe("Room.recalculate => Stripped State Events", function() {
@@ -701,8 +666,8 @@ describe("Room", function() {
            "room is an invite room", function() {
                const roomName = "flibble";

-                addMember(userA, "invite");
-                stateLookup["m.room.member$" + userA].event.invite_room_state = [
+                const event = addMember(userA, "invite");
+                event.event.invite_room_state = [
                    {
                        type: "m.room.name",
                        state_key: "",
@@ -712,30 +677,108 @@ describe("Room", function() {
                    },
                ];

-                room.recalculate(userA);
-                expect(room.currentState.setStateEvents).toHaveBeenCalled();
-                // first call, first arg (which is an array), first element in array
-                const fakeEvent = room.currentState.setStateEvents.calls[0].
-                      arguments[0][0];
-                expect(fakeEvent.getContent()).toEqual({
-                    name: roomName,
-                });
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
            });

            it("should not clobber state events if it isn't an invite room", function() {
-                addMember(userA, "join");
-                stateLookup["m.room.member$" + userA].event.invite_room_state = [
+                const event = addMember(userA, "join");
+                const roomName = "flibble";
+                setRoomName(roomName);
+                const roomNameToIgnore = "ignoreme";
+                event.event.invite_room_state = [
                    {
                        type: "m.room.name",
                        state_key: "",
                        content: {
-                            name: "flibble",
+                            name: roomNameToIgnore,
                        },
                    },
                ];

-                room.recalculate(userA);
-                expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
+            });
+        });
+
+        describe("Room.recalculate => Room Name using room summary", function() {
+            it("should use room heroes if available", function() {
+                addMember(userA, "invite");
+                addMember(userB);
+                addMember(userC);
+                addMember(userD);
+                room.setSummary({
+                    "m.heroes": [userB, userC, userD],
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(`${userB} and 2 others`);
+            });
+
+            it("missing hero member state reverts to mxid", function() {
+                room.setSummary({
+                    "m.heroes": [userB],
+                    "m.joined_member_count": 2,
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(userB);
+            });
+
+            it("uses hero name from state", function() {
+                const name = "Mr B";
+                addMember(userA, "invite");
+                addMember(userB, "join", {name});
+                room.setSummary({
+                    "m.heroes": [userB],
+                });
+
+                room.recalculate();
+                expect(room.name).toEqual(name);
+            });
+
+            it("uses counts from summary", function() {
+                const name = "Mr B";
+                addMember(userB, "join", {name});
+                room.setSummary({
+                    "m.heroes": [userB],
+                    "m.joined_member_count": 50,
+                    "m.invited_member_count": 50,
+                });
+                room.recalculate();
+                expect(room.name).toEqual(`${name} and 98 others`);
+            });
+
+            it("relies on heroes in case of absent counts", function() {
+                const nameB = "Mr Bean";
+                const nameC = "Mel C";
+                addMember(userB, "join", {name: nameB});
+                addMember(userC, "join", {name: nameC});
+                room.setSummary({
+                    "m.heroes": [userB, userC],
+                });
+                room.recalculate();
+                expect(room.name).toEqual(`${nameB} and ${nameC}`);
+            });
+
+            it("uses only heroes", function() {
+                const nameB = "Mr Bean";
+                addMember(userB, "join", {name: nameB});
+                addMember(userC, "join");
+                room.setSummary({
+                    "m.heroes": [userB],
+                });
+                room.recalculate();
+                expect(room.name).toEqual(nameB);
+            });
+
+            it("reverts to empty room in case of self chat", function() {
+                room.setSummary({
+                    "m.heroes": [],
+                    "m.invited_member_count": 1,
+                });
+                room.recalculate();
+                expect(room.name).toEqual("Empty room");
            });
        });

@@ -748,7 +791,7 @@ describe("Room", function() {
                addMember(userB);
                addMember(userC);
                addMember(userD);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                // we expect at least 1 member to be mentioned
                const others = [userB, userC, userD];
@@ -769,7 +812,7 @@ describe("Room", function() {
                addMember(userA);
                addMember(userB);
                addMember(userC);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
                expect(name.indexOf(userC)).toNotEqual(-1, name);
@@ -782,7 +825,7 @@ describe("Room", function() {
                addMember(userA);
                addMember(userB);
                addMember(userC);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
                expect(name.indexOf(userC)).toNotEqual(-1, name);
@@ -794,7 +837,7 @@ describe("Room", function() {
                setJoinRule("public");
                addMember(userA);
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -805,7 +848,7 @@ describe("Room", function() {
                setJoinRule("invite");
                addMember(userA);
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -815,7 +858,7 @@ describe("Room", function() {
                setJoinRule("invite");
                addMember(userA, "invite", {user: userB});
                addMember(userB);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name.indexOf(userB)).toNotEqual(-1, name);
            });
@@ -825,7 +868,7 @@ describe("Room", function() {
                const alias = "#room_alias:here";
                setJoinRule("invite");
                setAliases([alias, "#another:one"]);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
            });
@@ -835,7 +878,7 @@ describe("Room", function() {
                const alias = "#room_alias:here";
                setJoinRule("public");
                setAliases([alias, "#another:one"]);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(alias);
            });
@@ -845,7 +888,7 @@ describe("Room", function() {
                const roomName = "A mighty name indeed";
                setJoinRule("invite");
                setRoomName(roomName);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual(roomName);
            });
@@ -855,25 +898,23 @@ describe("Room", function() {
                const roomName = "A mighty name indeed";
                setJoinRule("public");
                setRoomName(roomName);
-                room.recalculate(userA);
-                const name = room.name;
-                expect(name).toEqual(roomName);
+                room.recalculate();
+                expect(room.name).toEqual(roomName);
            });

            it("should return 'Empty room' for private (invite join_rules) rooms if" +
            " a room name and alias don't exist and it is a self-chat.", function() {
                setJoinRule("invite");
                addMember(userA);
-                room.recalculate(userA);
-                const name = room.name;
-                expect(name).toEqual("Empty room");
+                room.recalculate();
+                expect(room.name).toEqual("Empty room");
            });

            it("should return 'Empty room' for public (public join_rules) rooms if a" +
            " room name and alias don't exist and it is a self-chat.", function() {
                setJoinRule("public");
                addMember(userA);
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual("Empty room");
            });
@@ -881,30 +922,30 @@ describe("Room", function() {
            it("should return 'Empty room' if there is no name, " +
               "alias or members in the room.",
            function() {
-                room.recalculate(userA);
+                room.recalculate();
                const name = room.name;
                expect(name).toEqual("Empty room");
            });

-            it("should return 'Invite from [inviter display name] if state event " +
+            it("should return '[inviter display name] if state event " +
               "available",
            function() {
                setJoinRule("invite");
-                addMember(userA, 'join', {name: "Alice"});
-                addMember(userB, "invite", {user: userA});
-                room.recalculate(userB);
+                addMember(userB, 'join', {name: "Alice"});
+                addMember(userA, "invite", {user: userA});
+                room.recalculate();
                const name = room.name;
-                expect(name).toEqual("Invite from Alice");
+                expect(name).toEqual("Alice");
            });

            it("should return inviter mxid if display name not available",
            function() {
                setJoinRule("invite");
-                addMember(userA);
-                addMember(userB, "invite", {user: userA});
-                room.recalculate(userB);
+                addMember(userB);
+                addMember(userA, "invite", {user: userA});
+                room.recalculate();
                const name = room.name;
-                expect(name).toEqual("Invite from " + userA);
+                expect(name).toEqual(userB);
            });
        });
    });
@@ -1151,7 +1192,7 @@ describe("Room", function() {
    describe("addPendingEvent", function() {
        it("should add pending events to the pendingEventList if " +
                      "pendingEventOrdering == 'detached'", function() {
-            const room = new Room(roomId, {
+            const room = new Room(roomId, null, userA, {
                pendingEventOrdering: "detached",
            });
            const eventA = utils.mkMessage({
@@ -1177,7 +1218,7 @@ describe("Room", function() {

        it("should add pending events to the timeline if " +
                      "pendingEventOrdering == 'chronological'", function() {
-            room = new Room(roomId, {
+            room = new Room(roomId, null, userA, {
                pendingEventOrdering: "chronological",
            });
            const eventA = utils.mkMessage({
@@ -1201,7 +1242,7 @@ describe("Room", function() {

    describe("updatePendingEvent", function() {
        it("should remove cancelled events from the pending list", function() {
-            const room = new Room(roomId, {
+            const room = new Room(roomId, null, userA, {
                pendingEventOrdering: "detached",
            });
            const eventA = utils.mkMessage({
@@ -1237,7 +1278,7 @@ describe("Room", function() {


        it("should remove cancelled events from the timeline", function() {
-            const room = new Room(roomId);
+            const room = new Room(roomId, null, userA);
            const eventA = utils.mkMessage({
                room: roomId, user: userA, event: true,
            });
@@ -1269,4 +1310,153 @@ describe("Room", function() {
            expect(callCount).toEqual(1);
        });
    });
+
+    describe("loadMembersIfNeeded", function() {
+        function createClientMock(serverResponse, storageResponse = null) {
+            return {
+                getEventMapper: function() {
+                    // events should already be MatrixEvents
+                    return function(event) {return event;};
+                },
+                isRoomEncrypted: function() {
+                    return false;
+                },
+                _http: {
+                    serverResponse,
+                    authedRequest: function() {
+                        if (this.serverResponse instanceof Error) {
+                            return Promise.reject(this.serverResponse);
+                        } else {
+                            return Promise.resolve({chunk: this.serverResponse});
+                        }
+                    },
+                },
+                store: {
+                    storageResponse,
+                    storedMembers: null,
+                    getOutOfBandMembers: function() {
+                        if (this.storageResponse instanceof Error) {
+                            return Promise.reject(this.storageResponse);
+                        } else {
+                            return Promise.resolve(this.storageResponse);
+                        }
+                    },
+                    setOutOfBandMembers: function(roomId, memberEvents) {
+                        this.storedMembers = memberEvents;
+                        return Promise.resolve();
+                    },
+                    getSyncToken: () => "sync_token",
+                },
+            };
+        }
+
+        const memberEvent = utils.mkMembership({
+            user: "@user_a:bar", mship: "join",
+            room: roomId, event: true, name: "User A",
+        });
+
+        it("should load members from server on first call", async function() {
+            const client = createClientMock([memberEvent]);
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            await room.loadMembersIfNeeded();
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+            const storedMembers = client.store.storedMembers;
+            expect(storedMembers.length).toEqual(1);
+            expect(storedMembers[0].event_id).toEqual(memberEvent.getId());
+        });
+
+        it("should take members from storage if available", async function() {
+            const memberEvent2 = utils.mkMembership({
+                user: "@user_a:bar", mship: "join",
+                room: roomId, event: true, name: "Ms A",
+            });
+            const client = createClientMock([memberEvent2], [memberEvent]);
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+
+            await room.loadMembersIfNeeded();
+
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+        });
+
+        it("should allow retry on error", async function() {
+            const client = createClientMock(new Error("server says no"));
+            const room = new Room(roomId, client, null, {lazyLoadMembers: true});
+            let hasThrown = false;
+            try {
+                await room.loadMembersIfNeeded();
+            } catch(err) {
+                hasThrown = true;
+            }
+            expect(hasThrown).toEqual(true);
+
+            client._http.serverResponse = [memberEvent];
+            await room.loadMembersIfNeeded();
+            const memberA = room.getMember("@user_a:bar");
+            expect(memberA.name).toEqual("User A");
+        });
+    });
+
+    describe("getMyMembership", function() {
+        it("should return synced membership if membership isn't available yet",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.updateMyMembership("invite");
+            expect(room.getMyMembership()).toEqual("invite");
+        });
+        it("should emit a Room.myMembership event on a change",
+        function() {
+            const room = new Room(roomId, null, userA);
+            const events = [];
+            room.on("Room.myMembership", (_room, membership, oldMembership) => {
+                events.push({membership, oldMembership});
+            });
+            room.updateMyMembership("invite");
+            expect(room.getMyMembership()).toEqual("invite");
+            expect(events[0]).toEqual({membership: "invite", oldMembership: null});
+            events.splice(0);   //clear
+            room.updateMyMembership("invite");
+            expect(events.length).toEqual(0);
+            room.updateMyMembership("join");
+            expect(room.getMyMembership()).toEqual("join");
+            expect(events[0]).toEqual({membership: "join", oldMembership: "invite"});
+        });
+    });
+
+    describe("guessDMUserId", function() {
+        it("should return first hero id",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.setSummary({'m.heroes': [userB]});
+            expect(room.guessDMUserId()).toEqual(userB);
+        });
+        it("should return first member that isn't self",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.addLiveEvents([utils.mkMembership({
+                user: userB, mship: "join",
+                room: roomId, event: true,
+            })]);
+            expect(room.guessDMUserId()).toEqual(userB);
+        });
+        it("should return self if only member present",
+        function() {
+            const room = new Room(roomId, null, userA);
+            expect(room.guessDMUserId()).toEqual(userA);
+        });
+    });
+
+    describe("maySendMessage", function() {
+        it("should return false if synced membership not join",
+        function() {
+            const room = new Room(roomId, null, userA);
+            room.updateMyMembership("invite");
+            expect(room.maySendMessage()).toEqual(false);
+            room.updateMyMembership("leave");
+            expect(room.maySendMessage()).toEqual(false);
+            room.updateMyMembership("join");
+            expect(room.maySendMessage()).toEqual(true);
+        });
+    });
 });
@@ -52,6 +52,11 @@ describe("SyncAccumulator", function() {
                                member("bob", "join"),
                            ],
                        },
+                        summary: {
+                            "m.heroes": undefined,
+                            "m.joined_member_count": undefined,
+                            "m.invited_member_count": undefined,
+                        },
                        timeline: {
                            events: [msg("alice", "hi")],
                            prev_batch: "something",
@@ -318,6 +323,58 @@ describe("SyncAccumulator", function() {
            },
        });
    });
+
+    describe("summary field", function() {
+        function createSyncResponseWithSummary(summary) {
+            return {
+                next_batch: "abc",
+                rooms: {
+                    invite: {},
+                    leave: {},
+                    join: {
+                        "!foo:bar": {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            unread_notifications: {},
+                            state: {
+                                events: [],
+                            },
+                            summary: summary,
+                            timeline: {
+                                events: [],
+                                prev_batch: "something",
+                            },
+                        },
+                    },
+                },
+            };
+        }
+
+        it("should copy summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.heroes"]).toEqual(["@alice:bar"]);
+        });
+
+        it("should accumulate summary properties", function() {
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@alice:bar"],
+                "m.invited_member_count": 2,
+            }));
+            sa.accumulate(createSyncResponseWithSummary({
+                "m.heroes": ["@bob:bar"],
+                "m.joined_member_count": 5,
+            }));
+            const summary = sa.getJSON().roomsData.join["!foo:bar"].summary;
+            expect(summary["m.invited_member_count"]).toEqual(2);
+            expect(summary["m.joined_member_count"]).toEqual(5);
+            expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
+        });
+    });
 });

 function syncSkeleton(joinObj) {
@@ -89,9 +89,14 @@ MatrixBaseApis.prototype.getHomeserverUrl = function() {

 /**
 * Get the Identity Server URL of this client
+ * @param {boolean} stripProto whether or not to strip the protocol from the URL
 * @return {string} Identity Server URL of this client
 */
-MatrixBaseApis.prototype.getIdentityServerUrl = function() {
+MatrixBaseApis.prototype.getIdentityServerUrl = function(stripProto=false) {
+    if (stripProto && (this.idBaseUrl.startsWith("http://") ||
+            this.idBaseUrl.startsWith("https://"))) {
+        return this.idBaseUrl.split("://")[1];
+    }
    return this.idBaseUrl;
 };

@@ -257,7 +262,19 @@ MatrixBaseApis.prototype.login = function(loginType, data, callback) {
    utils.extend(login_data, data);

    return this._http.authedRequest(
-        callback, "POST", "/login", undefined, login_data,
+        (error, response) => {
+            if (loginType === "m.login.password" && response &&
+                response.access_token && response.user_id) {
+                this._http.opts.accessToken = response.access_token;
+                this.credentials = {
+                    userId: response.user_id,
+                };
+            }
+
+            if (callback) {
+                callback(error, response);
+            }
+        }, "POST", "/login", undefined, login_data,
    );
 };

@@ -333,18 +350,29 @@ MatrixBaseApis.prototype.logout = function(callback) {
 * it is up to the caller to either reset or destroy the MatrixClient after
 * this method succeeds.
 * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
- * @param {module:client.callback} callback Optional.
+ * @param {boolean} erase Optional. If set, send as `erase` attribute in the
+ * JSON request body, indicating whether the account should be erased. Defaults
+ * to false.
 * @return {module:client.Promise} Resolves: On success, the empty object
 */
-MatrixBaseApis.prototype.deactivateAccount = function(auth, callback) {
-    let body = {};
-    if (auth) {
-        body = {
-            auth: auth,
-        };
+MatrixBaseApis.prototype.deactivateAccount = function(auth, erase) {
+    if (typeof(erase) === 'function') {
+        throw new Error(
+            'deactivateAccount no longer accepts a callback parameter',
+        );
    }
+
+    const body = {};
+    if (auth) {
+        body.auth = auth;
+    }
+    if (erase !== undefined) {
+        body.erase = erase;
+    }
+
    return this._http.authedRequestWithPrefix(
-        callback, "POST", '/account/deactivate', undefined, body, httpApi.PREFIX_UNSTABLE,
+        undefined, "POST", '/account/deactivate', undefined, body,
+        httpApi.PREFIX_R0,
    );
 };

@@ -401,6 +429,69 @@ MatrixBaseApis.prototype.roomState = function(roomId, callback) {
    return this._http.authedRequest(callback, "GET", path);
 };

+/**
+ * Get an event in a room by its event id.
+ * @param {string} roomId
+ * @param {string} eventId
+ * @param {module:client.callback} callback Optional.
+ *
+ * @return {Promise} Resolves to an object containing the event.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.fetchRoomEvent = function(roomId, eventId, callback) {
+    const path = utils.encodeUri(
+        "/rooms/$roomId/event/$eventId", {
+            $roomId: roomId,
+            $eventId: eventId,
+        },
+    );
+    return this._http.authedRequest(callback, "GET", path);
+};
+
+/**
+ * @param {string} roomId
+ * @param {string} includeMembership the membership type to include in the response
+ * @param {string} excludeMembership the membership type to exclude from the response
+ * @param {string} atEventId the id of the event for which moment in the timeline the members should be returned for
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: dictionary of userid to profile information
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.members =
+function(roomId, includeMembership, excludeMembership, atEventId, callback) {
+    const queryParams = {};
+    if (includeMembership) {
+        queryParams.membership = includeMembership;
+    }
+    if (excludeMembership) {
+        queryParams.not_membership = excludeMembership;
+    }
+    if (atEventId) {
+        queryParams.at = atEventId;
+    }
+
+    const queryString = utils.encodeParams(queryParams);
+
+    const path = utils.encodeUri("/rooms/$roomId/members?" + queryString,
+        {$roomId: roomId});
+    return this._http.authedRequest(callback, "GET", path);
+};
+
+/**
+ * Upgrades a room to a new protocol version
+ * @param {string} roomId
+ * @param {string} newVersion The target version to upgrade to
+ * @return {module:client.Promise} Resolves: Object with key 'replacement_room'
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.upgradeRoom = function(roomId, newVersion) {
+    const path = utils.encodeUri("/rooms/$roomId/upgrade", {$roomId: roomId});
+    return this._http.authedRequest(
+        undefined, "POST", path, undefined, {new_version: newVersion},
+    );
+};
+
+
 /**
 * @param {string} groupId
 * @return {module:client.Promise} Resolves: Group summary object
@@ -438,6 +529,27 @@ MatrixBaseApis.prototype.setGroupProfile = function(groupId, profile) {
    );
 };

+/**
+ * @param {string} groupId
+ * @param {object} policy The join policy for the group. Must include at
+ *     least a 'type' field which is 'open' if anyone can join the group
+ *     the group without prior approval, or 'invite' if an invite is
+ *     required to join.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupJoinPolicy = function(groupId, policy) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/settings/m.join_policy",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(
+        undefined, "PUT", path, undefined, {
+            'm.join_policy': policy,
+        },
+    );
+};
+
 /**
 * @param {string} groupId
 * @return {module:client.Promise} Resolves: Group users list object
@@ -448,6 +560,26 @@ MatrixBaseApis.prototype.getGroupUsers = function(groupId) {
    return this._http.authedRequest(undefined, "GET", path);
 };

+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group users list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupInvitedUsers = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/invited_users", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Group rooms list object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getGroupRooms = function(groupId) {
+    const path = utils.encodeUri("/groups/$groupId/rooms", {$groupId: groupId});
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
 /**
 * @param {string} groupId
 * @param {string} userId
@@ -478,14 +610,146 @@ MatrixBaseApis.prototype.removeUserFromGroup = function(groupId, userId) {

 /**
 * @param {string} groupId
+ * @param {string} userId
+ * @param {string} roleId Optional.
 * @return {module:client.Promise} Resolves: Empty object
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
-MatrixBaseApis.prototype.acceptGroupInvite = function(groupId) {
+MatrixBaseApis.prototype.addUserToGroupSummary = function(groupId, userId, roleId) {
+    const path = utils.encodeUri(
+        roleId ?
+            "/groups/$groupId/summary/$roleId/users/$userId" :
+            "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $roleId: roleId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} userId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeUserFromGroupSummary = function(groupId, userId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/users/$userId",
+        {$groupId: groupId, $userId: userId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {string} categoryId Optional.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroupSummary = function(groupId, roomId, categoryId) {
+    const path = utils.encodeUri(
+        categoryId ?
+            "/groups/$groupId/summary/$categoryId/rooms/$roomId" :
+            "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $categoryId: categoryId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroupSummary = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/summary/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.addRoomToGroup = function(groupId, roomId, isPublic) {
+    if (isPublic === undefined) {
+        isPublic = true;
+    }
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { "m.visibility": { type: isPublic ? "public" : "private" } },
+    );
+};
+
+/**
+ * Configure the visibility of a room-group association.
+ * @param {string} groupId
+ * @param {string} roomId
+ * @param {bool} isPublic Whether the room-group association is visible to non-members
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.updateGroupRoomVisibility = function(groupId, roomId, isPublic) {
+    // NB: The /config API is generic but there's not much point in exposing this yet as synapse
+    //     is the only server to implement this. In future we should consider an API that allows
+    //     arbitrary configuration, i.e. "config/$configKey".
+
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId/config/m.visibility",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined,
+        { type: isPublic ? "public" : "private" },
+    );
+};
+
+/**
+ * @param {string} groupId
+ * @param {string} roomId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.removeRoomFromGroup = function(groupId, roomId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/admin/rooms/$roomId",
+        {$groupId: groupId, $roomId: roomId},
+    );
+    return this._http.authedRequest(undefined, "DELETE", path, undefined, {});
+};
+
+/**
+ * @param {string} groupId
+ * @param {Object} opts Additional options to send alongside the acceptance.
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.acceptGroupInvite = function(groupId, opts = null) {
    const path = utils.encodeUri(
        "/groups/$groupId/self/accept_invite",
        {$groupId: groupId},
    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, opts || {});
+};
+
+/**
+ * @param {string} groupId
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.joinGroup = function(groupId) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/join",
+        {$groupId: groupId},
+    );
    return this._http.authedRequest(undefined, "PUT", path, undefined, {});
 };

@@ -545,6 +809,22 @@ MatrixBaseApis.prototype.getPublicisedGroups = function(userIds) {
    );
 };

+/**
+ * @param {string} groupId
+ * @param {bool} isPublic Whether the user's membership of this group is made public
+ * @return {module:client.Promise} Resolves: Empty object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.setGroupPublicity = function(groupId, isPublic) {
+    const path = utils.encodeUri(
+        "/groups/$groupId/self/update_publicity",
+        {$groupId: groupId},
+    );
+    return this._http.authedRequest(undefined, "PUT", path, undefined, {
+        publicise: isPublic,
+    });
+};
+
 /**
 * Retrieve a state event.
 * @param {string} roomId
@@ -659,6 +939,28 @@ MatrixBaseApis.prototype.setRoomReadMarkersHttpRequest =
    );
 };

+/**
+ * @return {module:client.Promise} Resolves: A list of the user's current rooms
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedRooms = function() {
+    const path = utils.encodeUri("/joined_rooms");
+    return this._http.authedRequest(undefined, "GET", path);
+};
+
+/**
+ * Retrieve membership info. for a room.
+ * @param {string} roomId ID of the room to get membership for
+ * @return {module:client.Promise} Resolves: A list of currently joined users
+ *                                 and their profile data.
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.getJoinedRoomMembers = function(roomId) {
+    const path = utils.encodeUri("/rooms/$roomId/joined_members", {
+        $roomId: roomId,
+    });
+    return this._http.authedRequest(undefined, "GET", path);
+};

 // Room Directory operations
 // =========================
@@ -866,6 +1168,10 @@ MatrixBaseApis.prototype.searchUserDirectory = function(opts) {
 * @param {string=} opts.name   Name to give the file on the server. Defaults
 *   to <tt>file.name</tt>.
 *
+ * @param {boolean=} opts.includeFilename if false will not send the filename,
+ *   e.g for encrypted file uploads where filename leaks are undesirable.
+ *   Defaults to true.
+ *
 * @param {string=} opts.type   Content-type for the upload. Defaults to
 *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
 *
@@ -1075,6 +1381,27 @@ MatrixBaseApis.prototype.deleteDevice = function(device_id, auth) {
    );
 };

+/**
+ * Delete multiple device
+ *
+ * @param {string[]} devices IDs of the devices to delete
+ * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
+ * @return {module:client.Promise} Resolves: result object
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.deleteMultipleDevices = function(devices, auth) {
+    const body = {devices};
+
+    if (auth) {
+        body.auth = auth;
+    }
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "POST", "/delete_devices", undefined, body,
+        httpApi.PREFIX_UNSTABLE,
+    );
+};
+

 // Push operations
 // ===============
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -39,18 +40,21 @@ const Filter = require("./filter");
 const SyncApi = require("./sync");
 const MatrixBaseApis = require("./base-apis");
 const MatrixError = httpApi.MatrixError;
+const ContentHelpers = require("./content-helpers");

 import ReEmitter from './ReEmitter';
+import RoomList from './crypto/RoomList';
+
+import Crypto from './crypto';
+import { isCryptoAvailable } from './crypto';
+
+// Disable warnings for now: we use deprecated bluebird functions
+// and need to migrate, but they spam the console with warnings.
+Promise.config({warnings: false});
+

 const SCROLLBACK_DELAY_MS = 3000;
-let CRYPTO_ENABLED = false;
-
-try {
-    var Crypto = require("./crypto");
-    CRYPTO_ENABLED = true;
-} catch (e) {
-    console.warn("Unable to load crypto module: crypto will be disabled: " + e);
-}
+const CRYPTO_ENABLED = isCryptoAvailable();

 /**
 * Construct a Matrix Client. Only directly construct this if you want to use
@@ -125,6 +129,8 @@ function MatrixClient(opts) {

    MatrixBaseApis.call(this, opts);

+    this.olmVersion = null; // Populated after initCrypto is done
+
    this.reEmitter = new ReEmitter(this);

    this.store = opts.store || new StubStore();
@@ -175,9 +181,17 @@ function MatrixClient(opts) {
    this._cryptoStore = opts.cryptoStore;
    this._sessionStore = opts.sessionStore;

-    if (CRYPTO_ENABLED) {
-        this.olmVersion = Crypto.getOlmVersion();
-    }
+    this._forceTURN = opts.forceTURN || false;
+
+    // List of which rooms have encryption enabled: separate from crypto because
+    // we still want to know which rooms are encrypted even if crypto is disabled:
+    // we don't want to start sending unencrypted events to them.
+    this._roomList = new RoomList(this._cryptoStore, this._sessionStore);
+
+    // The pushprocessor caches useful things, so keep one and re-use it
+    this._pushProcessor = new PushProcessor(this);
+
+    this._serverSupportsLazyLoading = null;
 }
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
@@ -252,6 +266,16 @@ MatrixClient.prototype.supportsVoip = function() {
    return this._supportsVoip;
 };

+/**
+ * Set whether VoIP calls are forced to use only TURN
+ * candidates. This is the same as the forceTURN option
+ * when creating the client.
+ * @param {bool} forceTURN True to force use of TURN servers
+ */
+MatrixClient.prototype.setForceTURN = function(forceTURN) {
+    this._forceTURN = forceTURN;
+};
+
 /**
 * Get the current sync state.
 * @return {?string} the sync state, which may be null.
@@ -264,6 +288,21 @@ MatrixClient.prototype.getSyncState = function() {
    return this._syncApi.getSyncState();
 };

+/**
+ * Returns the additional data object associated with
+ * the current sync state, or null if there is no
+ * such data.
+ * Sync errors, if available, are put in the 'error' key of
+ * this object.
+ * @return {?Object}
+ */
+MatrixClient.prototype.getSyncStateData = function() {
+    if (!this._syncApi) {
+        return null;
+    }
+    return this._syncApi.getSyncStateData();
+};
+
 /**
 * Return whether the client is configured for a guest account.
 * @return {boolean} True if this is a guest access_token (or no token is supplied).
@@ -333,18 +372,18 @@ MatrixClient.prototype.setNotifTimelineSet = function(notifTimelineSet) {
 * successfully initialised.
 */
 MatrixClient.prototype.initCrypto = async function() {
-    if (this._crypto) {
-        console.warn("Attempt to re-initialise e2e encryption on MatrixClient");
-        return;
-    }
-
-    if (!CRYPTO_ENABLED) {
+    if (!isCryptoAvailable()) {
        throw new Error(
            `End-to-end encryption not supported in this js-sdk build: did ` +
                `you remember to load the olm library?`,
        );
    }

+    if (this._crypto) {
+        console.warn("Attempt to re-initialise e2e encryption on MatrixClient");
+        return;
+    }
+
    if (!this._sessionStore) {
        // this is temporary, the sessionstore is supposed to be going away
        throw new Error(`Cannot enable encryption: no sessionStore provided`);
@@ -354,6 +393,9 @@ MatrixClient.prototype.initCrypto = async function() {
        throw new Error(`Cannot enable encryption: no cryptoStore provided`);
    }

+    // initialise the list of encrypted rooms (whether or not crypto is enabled)
+    await this._roomList.init();
+
    const userId = this.getUserId();
    if (userId === null) {
        throw new Error(
@@ -374,15 +416,20 @@ MatrixClient.prototype.initCrypto = async function() {
        userId, this.deviceId,
        this.store,
        this._cryptoStore,
+        this._roomList,
    );

    this.reEmitter.reEmit(crypto, [
        "crypto.roomKeyRequest",
        "crypto.roomKeyRequestCancellation",
+        "crypto.warning",
    ]);

    await crypto.init();

+    this.olmVersion = Crypto.getOlmVersion();
+
+
    // if crypto initialisation was successful, tell it to attach its event
    // handlers.
    crypto.registerEventHandlers(this);
@@ -543,11 +590,10 @@ async function _setDeviceVerification(

 /**
 * Set the global override for whether the client should ever send encrypted
- * messages to unverified devices.  If false, it can still be overridden
- * per-room.  If true, it overrides the per-room settings.
+ * messages to unverified devices.  This provides the default for rooms which
+ * do not specify a value.
 *
- * @param {boolean} value whether to unilaterally blacklist all
- * unverified devices
+ * @param {boolean} value whether to blacklist all unverified devices by default
 */
 MatrixClient.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
    if (this._crypto === null) {
@@ -557,8 +603,7 @@ MatrixClient.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
 };

 /**
- * @return {boolean} whether to unilaterally blacklist all
- * unverified devices
+ * @return {boolean} whether to blacklist all unverified devices by default
 */
 MatrixClient.prototype.getGlobalBlacklistUnverifiedDevices = function() {
    if (this._crypto === null) {
@@ -598,6 +643,16 @@ MatrixClient.prototype.isEventSenderVerified = async function(event) {
    return device.isVerified();
 };

+/**
+ * Cancel a room key request for this event if one is ongoing and resend the
+ * request.
+ * @param  {MatrixEvent} event event of which to cancel and resend the room
+ *                            key request.
+ */
+MatrixClient.prototype.cancelAndResendEventRoomKeyRequest = function(event) {
+    event.cancelAndResendKeyRequest(this._crypto);
+};
+
 /**
 * Enable end-to-end encryption for a room.
 * @param {string} roomId The room ID to enable encryption in.
@@ -634,11 +689,22 @@ MatrixClient.prototype.isRoomEncrypted = function(roomId) {
    // we don't have an m.room.encrypted event, but that might be because
    // the server is hiding it from us. Check the store to see if it was
    // previously encrypted.
-    if (!this._sessionStore) {
-        return false;
-    }
+    return this._roomList.isRoomEncrypted(roomId);
+};

-    return Boolean(this._sessionStore.getEndToEndRoom(roomId));
+/**
+ * Forces the current outbound group session to be discarded such
+ * that another one will be created next time an event is sent.
+ *
+ * @param {string} roomId The ID of the room to discard the session for
+ *
+ * This should not normally be necessary.
+ */
+MatrixClient.prototype.forceDiscardSession = function(roomId) {
+    if (!this._crypto) {
+        throw new Error("End-to-End encryption disabled");
+    }
+    this._crypto.forceDiscardSession(roomId);
 };

 /**
@@ -689,12 +755,23 @@ MatrixClient.prototype.getGroup = function(groupId) {

 /**
 * Retrieve all known groups.
- * @return {Groups[]} A list of groups, or an empty list if there is no data store.
+ * @return {Group[]} A list of groups, or an empty list if there is no data store.
 */
 MatrixClient.prototype.getGroups = function() {
    return this.store.getGroups();
 };

+/**
+ * Get the config for the media repository.
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves with an object containing the config.
+ */
+MatrixClient.prototype.getMediaConfig = function(callback) {
+    return this._http.requestWithPrefix(
+        callback, "GET", "/config", undefined, undefined, httpApi.PREFIX_MEDIA_R0,
+    );
+};
+
 // Room ops
 // ========

@@ -718,6 +795,37 @@ MatrixClient.prototype.getRooms = function() {
    return this.store.getRooms();
 };

+/**
+ * Retrieve all rooms that should be displayed to the user
+ * This is essentially getRooms() with some rooms filtered out, eg. old versions
+ * of rooms that have been replaced or (in future) other rooms that have been
+ * marked at the protocol level as not to be displayed to the user.
+ * @return {Room[]} A list of rooms, or an empty list if there is no data store.
+ */
+MatrixClient.prototype.getVisibleRooms = function() {
+    const allRooms = this.store.getRooms();
+
+    const replacedRooms = new Set();
+    for (const r of allRooms) {
+        const createEvent = r.currentState.getStateEvents('m.room.create', '');
+        // invites are included in this list and we don't know their create events yet
+        if (createEvent) {
+            const predecessor = createEvent.getContent()['predecessor'];
+            if (predecessor && predecessor['room_id']) {
+                replacedRooms.add(predecessor['room_id']);
+            }
+        }
+    }
+
+    return allRooms.filter((r) => {
+        const tombstone = r.currentState.getStateEvents('m.room.tombstone', '');
+        if (tombstone && replacedRooms.has(r.roomId)) {
+            return false;
+        }
+        return true;
+    });
+};
+
 /**
 * Retrieve a user.
 * @param {string} userId The user ID to retrieve.
@@ -760,7 +868,6 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
 /**
 * Get account data event of given type for the current user.
 * @param {string} eventType The event type
- * @param {module:client.callback} callback Optional.
 * @return {?object} The contents of the given account data event
 */
 MatrixClient.prototype.getAccountData = function(eventType) {
@@ -811,6 +918,8 @@ MatrixClient.prototype.isUserIgnored = function(userId) {
 * </strong> Default: true.
 * @param {boolean} opts.inviteSignUrl If the caller has a keypair 3pid invite,
 *                                     the signing URL is passed in this parameter.
+ * @param {string[]} opts.viaServers The server names to try and join through in
+ *                                   addition to those that are automatically chosen.
 * @param {module:client.callback} callback Optional.
 * @return {module:client.Promise} Resolves: Room object.
 * @return {module:http-api.MatrixError} Rejects: with an error response.
@@ -839,6 +948,13 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) {
        );
    }

+    const queryString = {};
+    if (opts.viaServers) {
+        queryString["server_name"] = opts.viaServers;
+    }
+
+    const reqOpts = {qsStringifyOptions: {arrayFormat: 'repeat'}};
+
    const defer = Promise.defer();

    const self = this;
@@ -849,7 +965,8 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) {
        }

        const path = utils.encodeUri("/join/$roomid", { $roomid: roomIdOrAlias});
-        return self._http.authedRequest(undefined, "POST", path, undefined, data);
+        return self._http.authedRequest(
+            undefined, "POST", path, queryString, data, reqOpts);
    }).then(function(res) {
        const roomId = res.room_id;
        const syncApi = new SyncApi(self, self._clientOpts);
@@ -1069,6 +1186,13 @@ MatrixClient.prototype.sendEvent = function(roomId, eventType, content, txnId,
        room.addPendingEvent(localEvent, txnId);
    }

+    // addPendingEvent can change the state to NOT_SENT if it believes
+    // that there's other events that have failed. We won't bother to
+    // try sending the event if the state has changed as such.
+    if (localEvent.status === EventStatus.NOT_SENT) {
+        return Promise.reject(new Error("Event blocked by other events not yet sent"));
+    }
+
    return _sendEvent(this, room, localEvent, callback);
 };

@@ -1125,8 +1249,14 @@ function _sendEvent(client, room, event, callback) {
        console.error("Error sending event", err.stack || err);

        try {
-            _updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            // set the error on the event before we update the status:
+            // updating the status emits the event, so the state should be
+            // consistent at that point.
            event.error = err;
+            _updatePendingEventStatus(room, event, EventStatus.NOT_SENT);
+            // also put the event object on the error: the caller will need this
+            // to resend or cancel the event
+            err.event = event;

            if (callback) {
                callback(err);
@@ -1243,10 +1373,7 @@ MatrixClient.prototype.sendMessage = function(roomId, content, txnId, callback)
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendTextMessage = function(roomId, body, txnId, callback) {
-    const content = {
-         msgtype: "m.text",
-         body: body,
-    };
+    const content = ContentHelpers.makeTextMessage(body);
    return this.sendMessage(roomId, content, txnId, callback);
 };

@@ -1259,10 +1386,7 @@ MatrixClient.prototype.sendTextMessage = function(roomId, body, txnId, callback)
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendNotice = function(roomId, body, txnId, callback) {
-    const content = {
-         msgtype: "m.notice",
-         body: body,
-    };
+    const content = ContentHelpers.makeNotice(body);
    return this.sendMessage(roomId, content, txnId, callback);
 };

@@ -1275,10 +1399,7 @@ MatrixClient.prototype.sendNotice = function(roomId, body, txnId, callback) {
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendEmoteMessage = function(roomId, body, txnId, callback) {
-    const content = {
-         msgtype: "m.emote",
-         body: body,
-    };
+    const content = ContentHelpers.makeEmoteMessage(body);
    return this.sendMessage(roomId, content, txnId, callback);
 };

@@ -1307,6 +1428,32 @@ MatrixClient.prototype.sendImageMessage = function(roomId, url, info, text, call
    return this.sendMessage(roomId, content, callback);
 };

+/**
+ * @param {string} roomId
+ * @param {string} url
+ * @param {Object} info
+ * @param {string} text
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: TODO
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.sendStickerMessage = function(roomId, url, info, text, callback) {
+    if (utils.isFunction(text)) {
+        callback = text; text = undefined;
+    }
+    if (!text) {
+        text = "Sticker";
+    }
+    const content = {
+         url: url,
+         info: info,
+         body: text,
+    };
+    return this.sendEvent(
+        roomId, "m.sticker", content, callback, undefined,
+    );
+};
+
 /**
 * @param {string} roomId
 * @param {string} body
@@ -1316,12 +1463,7 @@ MatrixClient.prototype.sendImageMessage = function(roomId, url, info, text, call
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendHtmlMessage = function(roomId, body, htmlBody, callback) {
-    const content = {
-        msgtype: "m.text",
-        format: "org.matrix.custom.html",
-        body: body,
-        formatted_body: htmlBody,
-    };
+    const content = ContentHelpers.makeHtmlMessage(body, htmlBody);
    return this.sendMessage(roomId, content, callback);
 };

@@ -1334,12 +1476,7 @@ MatrixClient.prototype.sendHtmlMessage = function(roomId, body, htmlBody, callba
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendHtmlNotice = function(roomId, body, htmlBody, callback) {
-    const content = {
-        msgtype: "m.notice",
-        format: "org.matrix.custom.html",
-        body: body,
-        formatted_body: htmlBody,
-    };
+    const content = ContentHelpers.makeHtmlNotice(body, htmlBody);
    return this.sendMessage(roomId, content, callback);
 };

@@ -1352,12 +1489,7 @@ MatrixClient.prototype.sendHtmlNotice = function(roomId, body, htmlBody, callbac
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
 MatrixClient.prototype.sendHtmlEmote = function(roomId, body, htmlBody, callback) {
-    const content = {
-        msgtype: "m.emote",
-        format: "org.matrix.custom.html",
-        body: body,
-        formatted_body: htmlBody,
-    };
+    const content = ContentHelpers.makeHtmlEmote(body, htmlBody);
    return this.sendMessage(roomId, content, callback);
 };

@@ -1533,18 +1665,13 @@ MatrixClient.prototype.inviteByThreePid = function(roomId, medium, address, call
        { $roomId: roomId },
    );

-    let identityServerUrl = this.getIdentityServerUrl();
+    const identityServerUrl = this.getIdentityServerUrl(true);
    if (!identityServerUrl) {
        return Promise.reject(new MatrixError({
            error: "No supplied identity server URL",
            errcode: "ORG.MATRIX.JSSDK_MISSING_PARAM",
        }));
    }
-    if (identityServerUrl.indexOf("http://") === 0 ||
-            identityServerUrl.indexOf("https://") === 0) {
-        // this request must not have the protocol part because reasons
-        identityServerUrl = identityServerUrl.split("://")[1];
-    }

    return this._http.authedRequest(callback, "POST", path, undefined, {
        id_server: identityServerUrl,
@@ -1704,8 +1831,7 @@ function _membershipChange(client, roomId, userId, membership, reason, callback)
 */
 MatrixClient.prototype.getPushActionsForEvent = function(event) {
    if (!event.getPushActions()) {
-        const pushProcessor = new PushProcessor(this);
-        event.setPushActions(pushProcessor.actionsForEvent(event));
+        event.setPushActions(this._pushProcessor.actionsForEvent(event));
    }
    return event.getPushActions();
 };
@@ -1886,14 +2012,6 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
    // reduce the required number of events appropriately
    limit = limit - numAdded;

-    const path = utils.encodeUri(
-        "/rooms/$roomId/messages", {$roomId: room.roomId},
-    );
-    const params = {
-        from: room.oldState.paginationToken,
-        limit: limit,
-        dir: 'b',
-    };
    const defer = Promise.defer();
    info = {
        promise: defer.promise,
@@ -1903,9 +2021,17 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
    // wait for a time before doing this request
    // (which may be 0 in order not to special case the code paths)
    Promise.delay(timeToWaitMs).then(function() {
-        return self._http.authedRequest(callback, "GET", path, params);
+        return self._createMessagesRequest(
+            room.roomId,
+            room.oldState.paginationToken,
+            limit,
+            'b');
    }).done(function(res) {
        const matrixEvents = utils.map(res.chunk, _PojoToMatrixEventMapper(self));
+        if (res.state) {
+            const stateEvents = utils.map(res.state, _PojoToMatrixEventMapper(self));
+            room.currentState.setUnknownStateEvents(stateEvents);
+        }
        room.addEventsToTimeline(matrixEvents, true, room.getLiveTimeline());
        room.oldState.paginationToken = res.end;
        if (res.chunk.length === 0) {
@@ -1924,73 +2050,6 @@ MatrixClient.prototype.scrollback = function(room, limit, callback) {
    return defer.promise;
 };

-/**
- * Take an EventContext, and back/forward-fill results.
- *
- * @param {module:models/event-context.EventContext} eventContext  context
- *    object to be updated
- * @param {Object}  opts
- * @param {boolean} opts.backwards  true to fill backwards, false to go forwards
- * @param {boolean} opts.limit      number of events to request
- *
- * @return {module:client.Promise} Resolves: updated EventContext object
- * @return {Error} Rejects: with an error response.
- */
-MatrixClient.prototype.paginateEventContext = function(eventContext, opts) {
-    // TODO: we should implement a backoff (as per scrollback()) to deal more
-    // nicely with HTTP errors.
-    opts = opts || {};
-    const backwards = opts.backwards || false;
-
-    const token = eventContext.getPaginateToken(backwards);
-    if (!token) {
-        // no more results.
-        return Promise.reject(new Error("No paginate token"));
-    }
-
-    const dir = backwards ? 'b' : 'f';
-    const pendingRequest = eventContext._paginateRequests[dir];
-
-    if (pendingRequest) {
-        // already a request in progress - return the existing promise
-        return pendingRequest;
-    }
-
-    const path = utils.encodeUri(
-        "/rooms/$roomId/messages", {$roomId: eventContext.getEvent().getRoomId()},
-    );
-    const params = {
-        from: token,
-        limit: ('limit' in opts) ? opts.limit : 30,
-        dir: dir,
-    };
-
-    const self = this;
-    const promise =
-        self._http.authedRequest(undefined, "GET", path, params,
-    ).then(function(res) {
-        let token = res.end;
-        if (res.chunk.length === 0) {
-            token = null;
-        } else {
-            const matrixEvents = utils.map(res.chunk, self.getEventMapper());
-            if (backwards) {
-                // eventContext expects the events in timeline order, but
-                // back-pagination returns them in reverse order.
-                matrixEvents.reverse();
-            }
-            eventContext.addEvents(matrixEvents, backwards);
-        }
-        eventContext.setPaginateToken(token, backwards);
-        return eventContext;
-    }).finally(function() {
-        eventContext._paginateRequests[dir] = null;
-    });
-    eventContext._paginateRequests[dir] = promise;
-
-    return promise;
-};
-
 /**
 * Get an EventTimeline for the given event
 *
@@ -2024,11 +2083,16 @@ MatrixClient.prototype.getEventTimeline = function(timelineSet, eventId) {
        },
    );

+    let params = undefined;
+    if (this._clientOpts.lazyLoadMembers) {
+        params = {filter: JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER)};
+    }
+
    // TODO: we should implement a backoff (as per scrollback()) to deal more
    // nicely with HTTP errors.
    const self = this;
    const promise =
-        self._http.authedRequest(undefined, "GET", path,
+        self._http.authedRequest(undefined, "GET", path, params,
    ).then(function(res) {
        if (!res.event) {
            throw new Error("'event' not in '/context' result - homeserver too old?");
@@ -2055,6 +2119,9 @@ MatrixClient.prototype.getEventTimeline = function(timelineSet, eventId) {
            timeline.initialiseState(utils.map(res.state,
                                               self.getEventMapper()));
            timeline.getState(EventTimeline.FORWARDS).paginationToken = res.end;
+        } else {
+            const stateEvents = utils.map(res.state, self.getEventMapper());
+            timeline.getState(EventTimeline.BACKWARDS).setUnknownStateEvents(stateEvents);
        }
        timelineSet.addEventsToTimeline(matrixEvents, true, timeline, res.start);

@@ -2069,6 +2136,49 @@ MatrixClient.prototype.getEventTimeline = function(timelineSet, eventId) {
    return promise;
 };

+/**
+ * Makes a request to /messages with the appropriate lazy loading filter set.
+ * XXX: if we do get rid of scrollback (as it's not used at the moment),
+ * we could inline this method again in paginateEventTimeline as that would
+ * then be the only call-site
+ * @param {string} roomId
+ * @param {string} fromToken
+ * @param {number} limit the maximum amount of events the retrieve
+ * @param {string} dir 'f' or 'b'
+ * @param {Filter} timelineFilter the timeline filter to pass
+ * @return {Promise}
+ */
+MatrixClient.prototype._createMessagesRequest =
+function(roomId, fromToken, limit, dir, timelineFilter = undefined) {
+    const path = utils.encodeUri(
+        "/rooms/$roomId/messages", {$roomId: roomId},
+    );
+    if (limit === undefined) {
+        limit = 30;
+    }
+    const params = {
+        from: fromToken,
+        limit: limit,
+        dir: dir,
+    };
+
+    let filter = null;
+    if (this._clientOpts.lazyLoadMembers) {
+        // create a shallow copy of LAZY_LOADING_MESSAGES_FILTER,
+        // so the timelineFilter doesn't get written into it below
+        filter = Object.assign({}, Filter.LAZY_LOADING_MESSAGES_FILTER);
+    }
+    if (timelineFilter) {
+        // XXX: it's horrific that /messages' filter parameter doesn't match
+        // /sync's one - see https://matrix.org/jira/browse/SPEC-451
+        filter = filter || {};
+        Object.assign(filter, timelineFilter.getRoomTimelineFilterComponent());
+    }
+    if (filter) {
+        params.filter = JSON.stringify(filter);
+    }
+    return this._http.authedRequest(undefined, "GET", path, params);
+};

 /**
 * Take an EventTimeline, and back/forward-fill results.
@@ -2163,25 +2273,18 @@ MatrixClient.prototype.paginateEventTimeline = function(eventTimeline, opts) {
            throw new Error("Unknown room " + eventTimeline.getRoomId());
        }

-        path = utils.encodeUri(
-            "/rooms/$roomId/messages", {$roomId: eventTimeline.getRoomId()},
-        );
-        params = {
-            from: token,
-            limit: ('limit' in opts) ? opts.limit : 30,
-            dir: dir,
-        };
-
-        const filter = eventTimeline.getFilter();
-        if (filter) {
-            // XXX: it's horrific that /messages' filter parameter doesn't match
-            // /sync's one - see https://matrix.org/jira/browse/SPEC-451
-            params.filter = JSON.stringify(filter.getRoomTimelineFilterComponent());
-        }
-
-        promise =
-            this._http.authedRequest(undefined, "GET", path, params,
-        ).then(function(res) {
+        promise = this._createMessagesRequest(
+            eventTimeline.getRoomId(),
+            token,
+            opts.limit,
+            dir,
+            eventTimeline.getFilter());
+        promise.then(function(res) {
+            if (res.state) {
+                const roomState = eventTimeline.getState(dir);
+                const stateEvents = utils.map(res.state, self.getEventMapper());
+                roomState.setUnknownStateEvents(stateEvents);
+            }
            const token = res.end;
            const matrixEvents = utils.map(res.chunk, self.getEventMapper());
            eventTimeline.getTimelineSet()
@@ -2983,8 +3086,14 @@ MatrixClient.prototype.getTurnServers = function() {
 *
 * @param {Filter=} opts.filter The filter to apply to /sync calls. This will override
 * the opts.initialSyncLimit, which would normally result in a timeline limit filter.
+ *
+ * @param {Boolean=} opts.disablePresence True to perform syncing without automatically
+ * updating presence.
+ * @param {Boolean=} opts.lazyLoadMembers True to not load all membership events during
+ * initial sync but fetch them when needed by calling `loadOutOfBandMembers`
+ * This will override the filter option at this moment.
 */
-MatrixClient.prototype.startClient = function(opts) {
+MatrixClient.prototype.startClient = async function(opts) {
    if (this.clientRunning) {
        // client is already running.
        return;
@@ -3022,11 +3131,29 @@ MatrixClient.prototype.startClient = function(opts) {
        return this._canResetTimelineCallback(roomId);
    };
    this._clientOpts = opts;
-
    this._syncApi = new SyncApi(this, opts);
    this._syncApi.sync();
 };

+/**
+ * store client options with boolean/string/numeric values
+ * to know in the next session what flags the sync data was
+ * created with (e.g. lazy loading)
+ * @param {object} opts the complete set of client options
+ * @return {Promise} for store operation */
+MatrixClient.prototype._storeClientOptions = function() {
+    const primTypes = ["boolean", "string", "number"];
+    const serializableOpts = Object.entries(this._clientOpts)
+        .filter(([key, value]) => {
+            return primTypes.includes(typeof value);
+        })
+        .reduce((obj, [key, value]) => {
+            obj[key] = value;
+            return obj;
+        }, {});
+    return this.store.storeClientOptions(serializableOpts);
+};
+
 /**
 * High level helper method to stop the client from polling and allow a
 * clean shutdown.
@@ -3049,6 +3176,36 @@ MatrixClient.prototype.stopClient = function() {
    global.clearTimeout(this._checkTurnServersTimeoutID);
 };

+/*
+ * Query the server to see if it support members lazy loading
+ * @return {Promise<boolean>} true if server supports lazy loading
+ */
+MatrixClient.prototype.doesServerSupportLazyLoading = async function() {
+    if (this._serverSupportsLazyLoading === null) {
+        const response = await this._http.request(
+            undefined, // callback
+            "GET", "/_matrix/client/versions",
+            undefined, // queryParams
+            undefined, // data
+            {
+                prefix: '',
+            },
+        );
+        const unstableFeatures = response["unstable_features"];
+        this._serverSupportsLazyLoading =
+            unstableFeatures && unstableFeatures["m.lazy_load_members"];
+    }
+    return this._serverSupportsLazyLoading;
+};
+
+/*
+ * Get if lazy loading members is being used.
+ * @return {boolean} Whether or not members are lazy loaded by this client
+ */
+MatrixClient.prototype.hasLazyLoadMembersEnabled = function() {
+    return !!this._clientOpts.lazyLoadMembers;
+};
+
 /*
 * Set a function which is called when /sync returns a 'limited' response.
 * It is called with a room ID and returns a boolean. It should return 'true' if the SDK
@@ -3097,10 +3254,7 @@ function setupCallEventHandler(client) {
            // now loop through the buffer chronologically and inject them
            callEventBuffer.forEach(function(e) {
                if (ignoreCallIds[e.getContent().call_id]) {
-                    console.log(
-                        'Ignoring previously answered/hungup call ' +
-                            e.getContent().call_id,
-                    );
+                    // This call has previously been ansered or hung up: ignore it
                    return;
                }
                callEventHandler(e);
@@ -3153,7 +3307,9 @@ function setupCallEventHandler(client) {
                );
            }

-            call = webRtcCall.createNewMatrixCall(client, event.getRoomId());
+            call = webRtcCall.createNewMatrixCall(client, event.getRoomId(), {
+                forceTURN: client._forceTURN,
+            });
            if (!call) {
                console.log(
                    "Incoming call ID " + content.call_id + " but this client " +
@@ -3395,6 +3551,12 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 * a state of SYNCING. <i>This is the equivalent of "syncComplete" in the
 * previous API.</i></li>
 *
+ * <li>CATCHUP: The client has detected the connection to the server might be
+ * available again and will now try to do a sync again. As this sync might take
+ * a long time (depending how long ago was last synced, and general server
+ * performance) the client is put in this mode so the UI can reflect trying
+ * to catch up with the server after losing connection.</li>
+ *
 * <li>SYNCING : The client is currently polling for new events from the server.
 * This will be called <i>after</i> processing latest events from a sync.</li>
 *
@@ -3418,11 +3580,11 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 *                                          +---->STOPPED
 *                                          |
 *              +----->PREPARED -------> SYNCING <--+
- *              |        ^                |  ^      |
- *              |        |                |  |      |
- *              |        |                V  |      |
- *   null ------+        |  +--------RECONNECTING   |
- *              |        |  V                       |
+ *              |                        ^  |  ^    |
+ *              |      CATCHUP ----------+  |  |    |
+ *              |        ^                  V  |    |
+ *   null ------+        |  +------- RECONNECTING   |
+ *              |        V  V                       |
 *              +------->ERROR ---------------------+
 *
 * NB: 'null' will never be emitted by this event.
@@ -3472,7 +3634,7 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 *
 * @param {?Object} data Data about this transition.
 *
- * @param {MatrixError} data.err The matrix error if <code>state=ERROR</code>.
+ * @param {MatrixError} data.error The matrix error if <code>state=ERROR</code>.
 *
 * @param {String} data.oldSyncToken The 'since' token passed to /sync.
 *    <code>null</code> for the first successful sync since this client was
@@ -3559,6 +3721,16 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 * });
 */

+/**
+ * Fires when the JS SDK receives a M_CONSENT_NOT_GIVEN error in response
+ * to a HTTP request.
+ * @event module:client~MatrixClient#"no_consent"
+ * @example
+ * matrixClient.on("no_consent", function(message, contentUri) {
+ *     console.info(message + ' Go to ' + contentUri);
+ * });
+ */
+
 /**
 * Fires when a device is marked as verified/unverified/blocked/unblocked by
 * {@link module:client~MatrixClient#setDeviceVerified|MatrixClient.setDeviceVerified} or
@@ -0,0 +1,100 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/** @module ContentHelpers */
+module.exports = {
+    /**
+     * Generates the content for a HTML Message event
+     * @param {string} body the plaintext body of the message
+     * @param {string} htmlBody the HTML representation of the message
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlMessage: function(body, htmlBody) {
+        return {
+            msgtype: "m.text",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Notice event
+     * @param {string} body the plaintext body of the notice
+     * @param {string} htmlBody the HTML representation of the notice
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlNotice: function(body, htmlBody) {
+        return {
+            msgtype: "m.notice",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a HTML Emote event
+     * @param {string} body the plaintext body of the emote
+     * @param {string} htmlBody the HTML representation of the emote
+     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+     */
+    makeHtmlEmote: function(body, htmlBody) {
+        return {
+            msgtype: "m.emote",
+            format: "org.matrix.custom.html",
+            body: body,
+            formatted_body: htmlBody,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Message event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeTextMessage: function(body) {
+        return {
+            msgtype: "m.text",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Notice event
+     * @param {string} body the plaintext body of the notice
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeNotice: function(body) {
+        return {
+            msgtype: "m.notice",
+            body: body,
+        };
+    },
+
+    /**
+     * Generates the content for a Plaintext Emote event
+     * @param {string} body the plaintext body of the emote
+     * @returns {{msgtype: string, body: string}}
+     */
+    makeEmoteMessage: function(body) {
+        return {
+            msgtype: "m.emote",
+            body: body,
+        };
+    },
+};
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -23,8 +24,10 @@ limitations under the License.

 import Promise from 'bluebird';

+import logger from '../logger';
 import DeviceInfo from './deviceinfo';
 import olmlib from './olmlib';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';


 /* State transition diagram for DeviceList._deviceTrackingStatus
@@ -58,31 +61,195 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
 * @alias module:crypto/DeviceList
 */
 export default class DeviceList {
-    constructor(baseApis, sessionStore, olmDevice) {
+    constructor(baseApis, cryptoStore, sessionStore, olmDevice) {
+        this._cryptoStore = cryptoStore;
        this._sessionStore = sessionStore;
-        this._serialiser = new DeviceListUpdateSerialiser(
-            baseApis, sessionStore, olmDevice,
-        );
+
+        // userId -> {
+        //     deviceId -> {
+        //         [device info]
+        //     }
+        // }
+        this._devices = {};

        // which users we are tracking device status for.
        // userId -> TRACKING_STATUS_*
-        this._deviceTrackingStatus = sessionStore.getEndToEndDeviceTrackingStatus() || {};
+        this._deviceTrackingStatus = {}; // loaded from storage in load()
+
+        // The 'next_batch' sync token at the point the data was writen,
+        // ie. a token representing the point immediately after the
+        // moment represented by the snapshot in the db.
+        this._syncToken = null;
+
+        this._serialiser = new DeviceListUpdateSerialiser(
+            baseApis, olmDevice, this,
+        );
+
+        // userId -> promise
+        this._keyDownloadsInProgressByUser = {};
+
+        // Set whenever changes are made other than setting the sync token
+        this._dirty = false;
+
+        // Promise resolved when device data is saved
+        this._savePromise = null;
+        // Function that resolves the save promise
+        this._resolveSavePromise = null;
+        // The time the save is scheduled for
+        this._savePromiseTime = null;
+        // The timer used to delay the save
+        this._saveTimer = null;
+    }
+
+    /**
+     * Load the device tracking state from storage
+     */
+    async load() {
+        let shouldDeleteSessionStore = false;
+        await this._cryptoStore.doTxn(
+            // migrate from session store if there's data there and not here
+            'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    if (deviceData === null) {
+                        logger.log("Migrating e2e device data...");
+                        this._devices = this._sessionStore.getAllEndToEndDevices() || {};
+                        this._deviceTrackingStatus = (
+                            this._sessionStore.getEndToEndDeviceTrackingStatus() || {}
+                        );
+                        this._syncToken = this._sessionStore.getEndToEndDeviceSyncToken();
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                        shouldDeleteSessionStore = true;
+                    } else {
+                        this._devices = deviceData ? deviceData.devices : {},
+                        this._deviceTrackingStatus = deviceData ?
+                            deviceData.trackingStatus : {};
+                        this._syncToken = deviceData ? deviceData.syncToken : null;
+                    }
+                });
+            },
+        );
+
+        if (shouldDeleteSessionStore) {
+            // migrated data is now safely persisted: remove from old store
+            this._sessionStore.removeEndToEndDeviceData();
+        }
+
        for (const u of Object.keys(this._deviceTrackingStatus)) {
            // if a download was in progress when we got shut down, it isn't any more.
            if (this._deviceTrackingStatus[u] == TRACKING_STATUS_DOWNLOAD_IN_PROGRESS) {
                this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
            }
        }
+    }

-        // userId -> promise
-        this._keyDownloadsInProgressByUser = {};
-
-        this.lastKnownSyncToken = null;
+    stop() {
+        if (this._saveTimer !== null) {
+            clearTimeout(this._saveTimer);
+        }
    }

    /**
-     * Download the keys for a list of users and stores the keys in the session
-     * store.
+     * Save the device tracking state to storage, if any changes are
+     * pending other than updating the sync token
+     *
+     * The actual save will be delayed by a short amount of time to
+     * aggregate multiple writes to the database.
+     *
+     * @param {integer} delay Time in ms before which the save actually happens.
+     *     By default, the save is delayed for a short period in order to batch
+     *     multiple writes, but this behaviour can be disabled by passing 0.
+     *
+     * @return {Promise<bool>} true if the data was saved, false if
+     *     it was not (eg. because no changes were pending). The promise
+     *     will only resolve once the data is saved, so may take some time
+     *     to resolve.
+     */
+    async saveIfDirty(delay) {
+        if (!this._dirty) return Promise.resolve(false);
+        // Delay saves for a bit so we can aggregate multiple saves that happen
+        // in quick succession (eg. when a whole room's devices are marked as known)
+        if (delay === undefined) delay = 500;
+
+        const targetTime = Date.now + delay;
+        if (this._savePromiseTime && targetTime < this._savePromiseTime) {
+            // There's a save scheduled but for after we would like: cancel
+            // it & schedule one for the time we want
+            clearTimeout(this._saveTimer);
+            this._saveTimer = null;
+            this._savePromiseTime = null;
+            // (but keep the save promise since whatever called save before
+            // will still want to know when the save is done)
+        }
+
+        let savePromise = this._savePromise;
+        if (savePromise === null) {
+            savePromise = new Promise((resolve, reject) => {
+                this._resolveSavePromise = resolve;
+            });
+            this._savePromise = savePromise;
+        }
+
+        if (this._saveTimer === null) {
+            const resolveSavePromise = this._resolveSavePromise;
+            this._savePromiseTime = targetTime;
+            this._saveTimer = setTimeout(() => {
+                logger.log('Saving device tracking data at token ' + this._syncToken);
+                // null out savePromise now (after the delay but before the write),
+                // otherwise we could return the existing promise when the save has
+                // actually already happened. Likewise for the dirty flag.
+                this._savePromiseTime = null;
+                this._saveTimer = null;
+                this._savePromise = null;
+                this._resolveSavePromise = null;
+
+                this._dirty = false;
+                this._cryptoStore.doTxn(
+                    'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
+                        this._cryptoStore.storeEndToEndDeviceData({
+                            devices: this._devices,
+                            trackingStatus: this._deviceTrackingStatus,
+                            syncToken: this._syncToken,
+                        }, txn);
+                    },
+                ).then(() => {
+                    resolveSavePromise();
+                });
+            }, delay);
+        }
+        return savePromise;
+    }
+
+    /**
+     * Gets the sync token last set with setSyncToken
+     *
+     * @return {string} The sync token
+     */
+    getSyncToken() {
+        return this._syncToken;
+    }
+
+    /**
+     * Sets the sync token that the app will pass as the 'since' to the /sync
+     * endpoint next time it syncs.
+     * The sync token must always be set after any changes made as a result of
+     * data in that sync since setting the sync token to a newer one will mean
+     * those changed will not be synced from the server if a new client starts
+     * up with that data.
+     *
+     * @param {string} st The sync token
+     */
+    setSyncToken(st) {
+        this._syncToken = st;
+    }
+
+    /**
+     * Ensures up to date keys for a list of users are stored in the session store,
+     * downloading and storing them if they're not (or if forceDownload is
+     * true).
     * @param {Array} userIds The users to fetch.
     * @param {bool} forceDownload Always download the keys even if cached.
     *
@@ -98,7 +265,7 @@ export default class DeviceList {
            if (this._keyDownloadsInProgressByUser[u]) {
                // already a key download in progress/queued for this user; its results
                // will be good enough for us.
-                console.log(
+                logger.log(
                    `downloadKeys: already have a download in progress for ` +
                    `${u}: awaiting its result`,
                );
@@ -109,13 +276,13 @@ export default class DeviceList {
        });

        if (usersToDownload.length != 0) {
-            console.log("downloadKeys: downloading for", usersToDownload);
+            logger.log("downloadKeys: downloading for", usersToDownload);
            const downloadPromise = this._doKeyDownload(usersToDownload);
            promises.push(downloadPromise);
        }

        if (promises.length === 0) {
-            console.log("downloadKeys: already have all necessary keys");
+            logger.log("downloadKeys: already have all necessary keys");
        }

        return Promise.all(promises).then(() => {
@@ -152,7 +319,7 @@ export default class DeviceList {
     * managed to get a list of devices for this user yet.
     */
    getStoredDevicesForUser(userId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
        if (!devs) {
            return null;
        }
@@ -165,6 +332,18 @@ export default class DeviceList {
        return res;
    }

+    /**
+     * Get the stored device data for a user, in raw object form
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @return {Object} deviceId->{object} devices, or undefined if
+     * there is no data for this user.
+     */
+    getRawStoredDevicesForUser(userId) {
+        return this._devices[userId];
+    }
+
    /**
     * Get the stored keys for a single device
     *
@@ -175,7 +354,7 @@ export default class DeviceList {
     * if we don't know about this device
     */
    getStoredDevice(userId, deviceId) {
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._devices[userId];
        if (!devs || !devs[deviceId]) {
            return undefined;
        }
@@ -200,7 +379,7 @@ export default class DeviceList {
            return null;
        }

-        const devices = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devices = this._devices[userId];
        if (!devices) {
            return null;
        }
@@ -229,6 +408,17 @@ export default class DeviceList {
        return null;
    }

+    /**
+     * Replaces the list of devices for a user with the given device list
+     *
+     * @param {string} u The user ID
+     * @param {Object} devs New device info for user
+     */
+    storeDevicesForUser(u, devs) {
+        this._devices[u] = devs;
+        this._dirty = true;
+    }
+
    /**
     * flag the given user for device-list tracking, if they are not already.
     *
@@ -250,12 +440,12 @@ export default class DeviceList {
            throw new Error('userId must be a string; was '+userId);
        }
        if (!this._deviceTrackingStatus[userId]) {
-            console.log('Now tracking device list for ' + userId);
+            logger.log('Now tracking device list for ' + userId);
            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
        }
        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
+        // of calls; we save all data together once the sync is done
+        this._dirty = true;
    }

    /**
@@ -269,14 +459,27 @@ export default class DeviceList {
     */
    stopTrackingDeviceList(userId) {
        if (this._deviceTrackingStatus[userId]) {
-            console.log('No longer tracking device list for ' + userId);
+            logger.log('No longer tracking device list for ' + userId);
            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
        }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
    }

+    /**
+     * Set all users we're currently tracking to untracked
+     *
+     * This will flag each user whose devices we are tracking as in need of an
+     * update.
+     */
+    stopTrackingAllDeviceLists() {
+        for (const userId of Object.keys(this._deviceTrackingStatus)) {
+            this._deviceTrackingStatus[userId] = TRACKING_STATUS_NOT_TRACKED;
+        }
+        this._dirty = true;
+    }

    /**
     * Mark the cached device list for the given user outdated.
@@ -291,23 +494,12 @@ export default class DeviceList {
     */
    invalidateUserDeviceList(userId) {
        if (this._deviceTrackingStatus[userId]) {
-            console.log("Marking device list outdated for", userId);
+            logger.log("Marking device list outdated for", userId);
            this._deviceTrackingStatus[userId] = TRACKING_STATUS_PENDING_DOWNLOAD;
-        }
-        // we don't yet persist the tracking status, since there may be a lot
-        // of calls; instead we wait for the forthcoming
-        // refreshOutdatedDeviceLists.
-    }

-    /**
-     * Mark all tracked device lists as outdated.
-     *
-     * This will flag each user whose devices we are tracking as in need of an
-     * update.
-     */
-    invalidateAllDeviceLists() {
-        for (const userId of Object.keys(this._deviceTrackingStatus)) {
-            this.invalidateUserDeviceList(userId);
+            // we don't yet persist the tracking status, since there may be a lot
+            // of calls; we save all data together once the sync is done
+            this._dirty = true;
        }
    }

@@ -318,6 +510,8 @@ export default class DeviceList {
     *    is no need to wait for this (it's mostly for the unit tests).
     */
    refreshOutdatedDeviceLists() {
+        this.saveIfDirty();
+
        const usersToDownload = [];
        for (const userId of Object.keys(this._deviceTrackingStatus)) {
            const stat = this._deviceTrackingStatus[userId];
@@ -326,13 +520,20 @@ export default class DeviceList {
            }
        }

-        // we didn't persist the tracking status during
-        // invalidateUserDeviceList, so do it now.
-        this._persistDeviceTrackingStatus();
-
        return this._doKeyDownload(usersToDownload);
    }

+    /**
+     * Set the stored device data for a user, in raw object form
+     * Used only by internal class DeviceListUpdateSerialiser
+     *
+     * @param {string} userId the user to get data for
+     *
+     * @param {Object} devices deviceId->{object} the new devices
+     */
+    _setRawStoredDevicesForUser(userId, devices) {
+        this._devices[userId] = devices;
+    }

    /**
     * Fire off download update requests for the given users, and update the
@@ -352,11 +553,11 @@ export default class DeviceList {
        }

        const prom = this._serialiser.updateDevicesForUsers(
-            users, this.lastKnownSyncToken,
+            users, this._syncToken,
        ).then(() => {
            finished(true);
        }, (e) => {
-            console.error(
+            logger.error(
                'Error downloading keys for ' + users + ":", e,
            );
            finished(false);
@@ -373,11 +574,13 @@ export default class DeviceList {

        const finished = (success) => {
            users.forEach((u) => {
+                this._dirty = true;
+
                // we may have queued up another download request for this user
                // since we started this request. If that happens, we should
                // ignore the completion of the first one.
                if (this._keyDownloadsInProgressByUser[u] !== prom) {
-                    console.log('Another update in the queue for', u,
+                    logger.log('Another update in the queue for', u,
                                '- not marking up-to-date');
                    return;
                }
@@ -388,21 +591,17 @@ export default class DeviceList {
                        // we didn't get any new invalidations since this download started:
                        // this user's device list is now up to date.
                        this._deviceTrackingStatus[u] = TRACKING_STATUS_UP_TO_DATE;
-                        console.log("Device list for", u, "now up to date");
+                        logger.log("Device list for", u, "now up to date");
                    } else {
                        this._deviceTrackingStatus[u] = TRACKING_STATUS_PENDING_DOWNLOAD;
                    }
                }
            });
-            this._persistDeviceTrackingStatus();
+            this.saveIfDirty();
        };

        return prom;
    }
-
-    _persistDeviceTrackingStatus() {
-        this._sessionStore.storeEndToEndDeviceTrackingStatus(this._deviceTrackingStatus);
-    }
 }

 /**
@@ -415,10 +614,15 @@ export default class DeviceList {
 * time (and queuing other requests up).
 */
 class DeviceListUpdateSerialiser {
-    constructor(baseApis, sessionStore, olmDevice) {
+    /*
+     * @param {object} baseApis Base API object
+     * @param {object} olmDevice The Olm Device
+     * @param {object} deviceList The device list object
+     */
+    constructor(baseApis, olmDevice, deviceList) {
        this._baseApis = baseApis;
-        this._sessionStore = sessionStore;
        this._olmDevice = olmDevice;
+        this._deviceList = deviceList; // the device list to be updated

        this._downloadInProgress = false;

@@ -431,9 +635,7 @@ class DeviceListUpdateSerialiser {
        // non-null indicates that we have users queued for download.
        this._queuedQueryDeferred = null;

-        // sync token to be used for the next query: essentially the
-        // most recent one we know about
-        this._nextSyncToken = null;
+        this._syncToken = null; // The sync token we send with the requests
    }

    /**
@@ -452,15 +654,19 @@ class DeviceListUpdateSerialiser {
        users.forEach((u) => {
            this._keyDownloadsQueuedByUser[u] = true;
        });
-        this._nextSyncToken = syncToken;

        if (!this._queuedQueryDeferred) {
            this._queuedQueryDeferred = Promise.defer();
        }

+        // We always take the new sync token and just use the latest one we've
+        // been given, since it just needs to be at least as recent as the
+        // sync response the device invalidation message arrived in
+        this._syncToken = syncToken;
+
        if (this._downloadInProgress) {
            // just queue up these users
-            console.log('Queued key download for', users);
+            logger.log('Queued key download for', users);
            return this._queuedQueryDeferred.promise;
        }

@@ -480,12 +686,12 @@ class DeviceListUpdateSerialiser {
        const deferred = this._queuedQueryDeferred;
        this._queuedQueryDeferred = null;

-        console.log('Starting key download for', downloadUsers);
+        logger.log('Starting key download for', downloadUsers);
        this._downloadInProgress = true;

        const opts = {};
-        if (this._nextSyncToken) {
-            opts.token = this._nextSyncToken;
+        if (this._syncToken) {
+            opts.token = this._syncToken;
        }

        this._baseApis.downloadKeysForUsers(
@@ -507,7 +713,7 @@ class DeviceListUpdateSerialiser {

            return prom;
        }).done(() => {
-            console.log('Completed key download for ' + downloadUsers);
+            logger.log('Completed key download for ' + downloadUsers);

            this._downloadInProgress = false;
            deferred.resolve();
@@ -517,8 +723,8 @@ class DeviceListUpdateSerialiser {
                this._doQueuedQueries();
            }
        }, (e) => {
-            console.warn('Error downloading keys for ' + downloadUsers + ':', e);
-            this._downloadInProgressInProgress = false;
+            logger.warn('Error downloading keys for ' + downloadUsers + ':', e);
+            this._downloadInProgress = false;
            deferred.reject(e);
        });

@@ -526,11 +732,11 @@ class DeviceListUpdateSerialiser {
    }

    async _processQueryResponseForUser(userId, response) {
-        console.log('got keys for ' + userId + ':', response);
+        logger.log('got keys for ' + userId + ':', response);

        // map from deviceid -> deviceinfo for this user
        const userStore = {};
-        const devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
        if (devs) {
            Object.keys(devs).forEach((deviceId) => {
                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
@@ -542,15 +748,13 @@ class DeviceListUpdateSerialiser {
            this._olmDevice, userId, userStore, response || {},
        );

-        // update the session store
+        // put the updates into thr object that will be returned as our results
        const storage = {};
        Object.keys(userStore).forEach((deviceId) => {
            storage[deviceId] = userStore[deviceId].toStorage();
        });

-        this._sessionStore.storeEndToEndDevicesForUser(
-            userId, storage,
-        );
+        this._deviceList._setRawStoredDevicesForUser(userId, storage);
    }
 }

@@ -566,7 +770,7 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        }

        if (!(deviceId in userResult)) {
-            console.log("Device " + userId + ":" + deviceId +
+            logger.log("Device " + userId + ":" + deviceId +
                " has been removed");
            delete userStore[deviceId];
            updated = true;
@@ -583,12 +787,12 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
        // check that the user_id and device_id in the response object are
        // correct
        if (deviceResult.user_id !== userId) {
-            console.warn("Mismatched user_id " + deviceResult.user_id +
+            logger.warn("Mismatched user_id " + deviceResult.user_id +
               " in keys from " + userId + ":" + deviceId);
            continue;
        }
        if (deviceResult.device_id !== deviceId) {
-            console.warn("Mismatched device_id " + deviceResult.device_id +
+            logger.warn("Mismatched device_id " + deviceResult.device_id +
               " in keys from " + userId + ":" + deviceId);
            continue;
        }
@@ -618,7 +822,7 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
    const signKeyId = "ed25519:" + deviceId;
    const signKey = deviceResult.keys[signKeyId];
    if (!signKey) {
-        console.warn("Device " + userId + ":" + deviceId +
+        logger.warn("Device " + userId + ":" + deviceId +
            " has no ed25519 key");
        return false;
    }
@@ -628,7 +832,7 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
    try {
        await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
    } catch (e) {
-        console.warn("Unable to verify signature on device " +
+        logger.warn("Unable to verify signature on device " +
            userId + ":" + deviceId + ":" + e);
        return false;
    }
@@ -645,7 +849,7 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
            // best off sticking with the original keys.
            //
            // Should we warn the user about it somehow?
-            console.warn("Ed25519 key for device " + userId + ":" +
+            logger.warn("Ed25519 key for device " + userId + ":" +
               deviceId + " has changed");
            return false;
        }
@@ -16,6 +16,7 @@ limitations under the License.

 import Promise from 'bluebird';

+import logger from '../logger';
 import utils from '../utils';

 /**
@@ -35,13 +36,19 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
 *
 * The state machine looks like:
 *
- *     |
- *     V         (cancellation requested)
- *   UNSENT  -----------------------------+
- *     |                                  |
- *     | (send successful)                |
- *     V                                  |
- *    SENT                                |
+ *     |         (cancellation sent)
+ *     | .-------------------------------------------------.
+ *     | |                                                 |
+ *     V V       (cancellation requested)                  |
+ *   UNSENT  -----------------------------+                |
+ *     |                                  |                |
+ *     |                                  |                |
+ *     | (send successful)                |  CANCELLATION_PENDING_AND_WILL_RESEND
+ *     V                                  |                Λ
+ *    SENT                                |                |
+ *     |--------------------------------  |  --------------'
+ *     |                                  |  (cancellation requested with intent
+ *     |                                  |   to resend the original request)
 *     |                                  |
 *     | (cancellation requested)         |
 *     V                                  |
@@ -62,6 +69,12 @@ const ROOM_KEY_REQUEST_STATES = {

    /** reply received, cancellation not yet sent */
    CANCELLATION_PENDING: 2,
+
+    /**
+     * Cancellation not yet sent and will transition to UNSENT instead of
+     * being deleted once the cancellation has been sent.
+     */
+    CANCELLATION_PENDING_AND_WILL_RESEND: 3,
 };

 export default class OutgoingRoomKeyRequestManager {
@@ -96,7 +109,7 @@ export default class OutgoingRoomKeyRequestManager {
     * Called when the client is stopped. Stops any running background processes.
     */
    stop() {
-        console.log('stopping OutgoingRoomKeyRequestManager');
+        logger.log('stopping OutgoingRoomKeyRequestManager');
        // stop the timer on the next run
        this._clientRunning = false;
    }
@@ -130,14 +143,16 @@ export default class OutgoingRoomKeyRequestManager {
    }

    /**
-     * Cancel room key requests, if any match the given details
+     * Cancel room key requests, if any match the given requestBody
     *
     * @param {module:crypto~RoomKeyRequestBody} requestBody
+     * @param {boolean} andResend if true, transition to UNSENT instead of
+     *                            deleting after sending cancellation.
     *
     * @returns {Promise} resolves when the request has been updated in our
     *    pending list.
     */
-    cancelRoomKeyRequest(requestBody) {
+    cancelRoomKeyRequest(requestBody, andResend=false) {
        return this._cryptoStore.getOutgoingRoomKeyRequest(
            requestBody,
        ).then((req) => {
@@ -147,6 +162,7 @@ export default class OutgoingRoomKeyRequestManager {
            }
            switch (req.state) {
                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
                    // nothing to do here
                    return;

@@ -158,7 +174,7 @@ export default class OutgoingRoomKeyRequestManager {
                    // may have seen it, so we still need to send a cancellation
                    // in that case :/

-                    console.log(
+                    logger.log(
                        'deleting unnecessary room key request for ' +
                        stringifyRequestBody(requestBody),
                    );
@@ -166,11 +182,16 @@ export default class OutgoingRoomKeyRequestManager {
                        req.requestId, ROOM_KEY_REQUEST_STATES.UNSENT,
                    );

-                case ROOM_KEY_REQUEST_STATES.SENT:
+                case ROOM_KEY_REQUEST_STATES.SENT: {
+                    // If `andResend` is set, transition to UNSENT once the
+                    // cancellation has successfully been sent.
+                    const state = andResend ?
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND :
+                        ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING;
                    // send a cancellation.
                    return this._cryptoStore.updateOutgoingRoomKeyRequest(
                        req.requestId, ROOM_KEY_REQUEST_STATES.SENT, {
-                            state: ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+                            state,
                            cancellationTxnId: this._baseApis.makeTxnId(),
                        },
                    ).then((updatedReq) => {
@@ -181,7 +202,7 @@ export default class OutgoingRoomKeyRequestManager {
                            // the request cancelled. There is no point in
                            // sending another cancellation since the other tab
                            // will do it.
-                            console.log(
+                            logger.log(
                                'Tried to cancel room key request for ' +
                                stringifyRequestBody(requestBody) +
                                ' but it was already cancelled in another tab',
@@ -200,15 +221,23 @@ export default class OutgoingRoomKeyRequestManager {
                        // do.)
                        this._sendOutgoingRoomKeyRequestCancellation(
                            updatedReq,
+                            andResend,
                        ).catch((e) => {
-                            console.error(
+                            logger.error(
                                "Error sending room key request cancellation;"
                                + " will retry later.", e,
                            );
                            this._startTimer();
-                        }).done();
+                        }).then(() => {
+                            if (!andResend) return;
+                            // The request has transitioned from
+                            // CANCELLATION_PENDING_AND_WILL_RESEND to UNSENT. We
+                            // still need to resend the request which is now UNSENT, so
+                            // start the timer if it isn't already started.
+                            this._startTimer();
+                        });
                    });
-
+                }
                default:
                    throw new Error('unhandled state: ' + req.state);
            }
@@ -233,7 +262,7 @@ export default class OutgoingRoomKeyRequestManager {
            }).catch((e) => {
                // this should only happen if there is an indexeddb error,
                // in which case we're a bit stuffed anyway.
-                console.warn(
+                logger.warn(
                    `error in OutgoingRoomKeyRequestManager: ${e}`,
                );
            }).done();
@@ -254,30 +283,37 @@ export default class OutgoingRoomKeyRequestManager {
            return Promise.resolve();
        }

-        console.log("Looking for queued outgoing room key requests");
+        logger.log("Looking for queued outgoing room key requests");

        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
+            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
            ROOM_KEY_REQUEST_STATES.UNSENT,
        ]).then((req) => {
            if (!req) {
-                console.log("No more outgoing room key requests");
+                logger.log("No more outgoing room key requests");
                this._sendOutgoingRoomKeyRequestsTimer = null;
                return;
            }

            let prom;
-            if (req.state === ROOM_KEY_REQUEST_STATES.UNSENT) {
-                prom = this._sendOutgoingRoomKeyRequest(req);
-            } else { // must be a cancellation
-                prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+            switch (req.state) {
+                case ROOM_KEY_REQUEST_STATES.UNSENT:
+                    prom = this._sendOutgoingRoomKeyRequest(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req);
+                    break;
+                case ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND:
+                    prom = this._sendOutgoingRoomKeyRequestCancellation(req, true);
+                    break;
            }

            return prom.then(() => {
                // go around the loop again
                return this._sendOutgoingRoomKeyRequests();
            }).catch((e) => {
-                console.error("Error sending room key request; will retry later.", e);
+                logger.error("Error sending room key request; will retry later.", e);
                this._sendOutgoingRoomKeyRequestsTimer = null;
                this._startTimer();
            }).done();
@@ -286,7 +322,7 @@ export default class OutgoingRoomKeyRequestManager {

    // given a RoomKeyRequest, send it and update the request record
    _sendOutgoingRoomKeyRequest(req) {
-        console.log(
+        logger.log(
            `Requesting keys for ${stringifyRequestBody(req.requestBody)}` +
            ` from ${stringifyRecipientList(req.recipients)}` +
            `(id ${req.requestId})`,
@@ -309,9 +345,10 @@ export default class OutgoingRoomKeyRequestManager {
        });
    }

-    // given a RoomKeyRequest, cancel it and delete the request record
-    _sendOutgoingRoomKeyRequestCancellation(req) {
-        console.log(
+    // Given a RoomKeyRequest, cancel it and delete the request record unless
+    // andResend is set, in which case transition to UNSENT.
+    _sendOutgoingRoomKeyRequestCancellation(req, andResend) {
+        logger.log(
            `Sending cancellation for key request for ` +
            `${stringifyRequestBody(req.requestBody)} to ` +
            `${stringifyRecipientList(req.recipients)} ` +
@@ -327,6 +364,14 @@ export default class OutgoingRoomKeyRequestManager {
        return this._sendMessageToDevices(
            requestMessage, req.recipients, req.cancellationTxnId,
        ).then(() => {
+            if (andResend) {
+                // We want to resend, so transition to UNSENT
+                return this._cryptoStore.updateOutgoingRoomKeyRequest(
+                    req.requestId,
+                    ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
+                    { state: ROOM_KEY_REQUEST_STATES.UNSENT },
+                );
+            }
            return this._cryptoStore.deleteOutgoingRoomKeyRequest(
                req.requestId, ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
            );
@@ -0,0 +1,84 @@
+/*
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module crypto/RoomList
+ *
+ * Manages the list of encrypted rooms
+ */
+
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+/**
+ * @alias module:crypto/RoomList
+ */
+export default class RoomList {
+    constructor(cryptoStore, sessionStore) {
+        this._cryptoStore = cryptoStore;
+        this._sessionStore = sessionStore;
+
+        // Object of roomId -> room e2e info object (body of the m.room.encryption event)
+        this._roomEncryption = {};
+    }
+
+    async init() {
+        let removeSessionStoreRooms = false;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.getEndToEndRooms(txn, (result) => {
+                    if (result === null || Object.keys(result).length === 0) {
+                        // migrate from session store, if there's data there
+                        const sessStoreRooms = this._sessionStore.getAllEndToEndRooms();
+                        if (sessStoreRooms !== null) {
+                            for (const roomId of Object.keys(sessStoreRooms)) {
+                                this._cryptoStore.storeEndToEndRoom(
+                                    roomId, sessStoreRooms[roomId], txn,
+                                );
+                            }
+                        }
+                        this._roomEncryption = sessStoreRooms;
+                        removeSessionStoreRooms = true;
+                    } else {
+                        this._roomEncryption = result;
+                    }
+                });
+            },
+        );
+        if (removeSessionStoreRooms) {
+            this._sessionStore.removeAllEndToEndRooms();
+        }
+    }
+
+    getRoomEncryption(roomId) {
+        return this._roomEncryption[roomId] || null;
+    }
+
+    isRoomEncrypted(roomId) {
+        return Boolean(this.getRoomEncryption(roomId));
+    }
+
+    async setRoomEncryption(roomId, roomInfo) {
+        // important that this happens before calling into the store
+        // as it prevents the Crypto::setRoomEncryption from calling
+        // this twice for consecutive m.room.encryption events
+        this._roomEncryption[roomId] = roomInfo;
+        await this._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ROOMS], (txn) => {
+                this._cryptoStore.storeEndToEndRoom(roomId, roomInfo, txn);
+            },
+        );
+    }
+}
@@ -176,34 +176,30 @@ export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 * @extends Error
 */
 class DecryptionError extends Error {
-    constructor(msg, details) {
+    constructor(code, msg, details) {
        super(msg);
+        this.code = code;
        this.name = 'DecryptionError';
-        this.details = details;
-    }
-
-    /**
-     * override the string used when logging
-     *
-     * @returns {String}
-     */
-    toString() {
-        let result = this.name + '[msg: ' + this.message;
-
-        if (this.details) {
-            result += ', ' +
-                Object.keys(this.details).map(
-                    (k) => k + ': ' + this.details[k],
-                ).join(', ');
-        }
-
-        result += ']';
-
-        return result;
+        this.detailedString = _detailedStringForDecryptionError(this, details);
    }
 }
 export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272

+function _detailedStringForDecryptionError(err, details) {
+    let result = err.name + '[msg: ' + err.message;
+
+    if (details) {
+        result += ', ' +
+            Object.keys(details).map(
+                (k) => k + ': ' + details[k],
+            ).join(', ');
+    }
+
+    result += ']';
+
+    return result;
+}
+
 /**
 * Exception thrown specifically when we want to warn the user to consider
 * the security of their conversation before continuing
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -23,6 +24,7 @@ limitations under the License.

 import Promise from 'bluebird';

+const logger = require("../../logger");
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
 const base = require("./base");
@@ -64,7 +66,7 @@ OutboundSessionInfo.prototype.needsRotation = function(
    if (this.useCount >= rotationPeriodMsgs ||
        sessionLifetime >= rotationPeriodMs
       ) {
-        console.log(
+        logger.log(
            "Rotating megolm session after " + this.useCount +
                " messages, " + sessionLifetime + "ms",
        );
@@ -74,6 +76,14 @@ OutboundSessionInfo.prototype.needsRotation = function(
    return false;
 };

+OutboundSessionInfo.prototype.markSharedWithDevice = function(
+    userId, deviceId, chainIndex,
+) {
+    if (!this.sharedWithDevices[userId]) {
+        this.sharedWithDevices[userId] = {};
+    }
+    this.sharedWithDevices[userId][deviceId] = chainIndex;
+};

 /**
 * Determine if this session has been shared with devices which it shouldn't
@@ -94,7 +104,7 @@ OutboundSessionInfo.prototype.sharedWithTooManyDevices = function(
        }

        if (!devicesInRoom.hasOwnProperty(userId)) {
-            console.log("Starting new session because we shared with " + userId);
+            logger.log("Starting new session because we shared with " + userId);
            return true;
        }

@@ -104,7 +114,7 @@ OutboundSessionInfo.prototype.sharedWithTooManyDevices = function(
            }

            if (!devicesInRoom[userId].hasOwnProperty(deviceId)) {
-                console.log(
+                logger.log(
                    "Starting new session because we shared with " +
                        userId + ":" + deviceId,
                );
@@ -173,7 +183,7 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
        if (session && session.needsRotation(self._sessionRotationPeriodMsgs,
                                             self._sessionRotationPeriodMs)
           ) {
-            console.log("Starting new megolm session because we need to rotate.");
+            logger.log("Starting new megolm session because we need to rotate.");
            session = null;
        }

@@ -183,7 +193,7 @@ MegolmEncryption.prototype._ensureOutboundSession = function(devicesInRoom) {
        }

        if (!session) {
-            console.log(`Starting new megolm session for room ${self._roomId}`);
+            logger.log(`Starting new megolm session for room ${self._roomId}`);
            session = await self._prepareNewSession();
        }

@@ -262,15 +272,151 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
 *
 * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
 *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceId>} devicemap
+ *   mapping from userId to deviceId to {@link module:crypto~OlmSessionResult}
+ *
 * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
 *    map from userid to list of devices
 *
- * @return {module:client.Promise} Promise which resolves once the key sharing
- *     message has been sent.
+ * @return {array<object<userid, deviceInfo>>}
 */
-MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUser) {
-    const self = this;
+MegolmEncryption.prototype._splitUserDeviceMap = function(
+    session, chainIndex, devicemap, devicesByUser,
+) {
+    const maxToDeviceMessagesPerRequest = 20;

+    // use an array where the slices of a content map gets stored
+    const mapSlices = [];
+    let currentSliceId = 0; // start inserting in the first slice
+    let entriesInCurrentSlice = 0;
+
+    for (const userId of Object.keys(devicesByUser)) {
+        const devicesToShareWith = devicesByUser[userId];
+        const sessionResults = devicemap[userId];
+
+        for (let i = 0; i < devicesToShareWith.length; i++) {
+            const deviceInfo = devicesToShareWith[i];
+            const deviceId = deviceInfo.deviceId;
+
+            const sessionResult = sessionResults[deviceId];
+            if (!sessionResult.sessionId) {
+                // no session with this device, probably because there
+                // were no one-time keys.
+                //
+                // we could send them a to_device message anyway, as a
+                // signal that they have missed out on the key sharing
+                // message because of the lack of keys, but there's not
+                // much point in that really; it will mostly serve to clog
+                // up to_device inboxes.
+
+                // mark this device as "handled" because we don't want to try
+                // to claim a one-time-key for dead devices on every message.
+                session.markSharedWithDevice(userId, deviceId, chainIndex);
+
+                // ensureOlmSessionsForUsers has already done the logging,
+                // so just skip it.
+                continue;
+            }
+
+            logger.log(
+                "share keys with device " + userId + ":" + deviceId,
+            );
+
+            if (entriesInCurrentSlice > maxToDeviceMessagesPerRequest) {
+                // the current slice is filled up. Start inserting into the next slice
+                entriesInCurrentSlice = 0;
+                currentSliceId++;
+            }
+            if (!mapSlices[currentSliceId]) {
+                mapSlices[currentSliceId] = [];
+            }
+
+            mapSlices[currentSliceId].push({
+                userId: userId,
+                deviceInfo: deviceInfo,
+            });
+
+            entriesInCurrentSlice++;
+        }
+    }
+    return mapSlices;
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {number} chainIndex current chain index
+ *
+ * @param {object<userId, deviceInfo>} userDeviceMap
+ *   mapping from userId to deviceInfo
+ *
+ * @param {object} payload fields to include in the encrypted payload
+ *
+ * @return {module:client.Promise} Promise which resolves once the key sharing
+ *     for the given userDeviceMap is generated and has been sent.
+ */
+MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
+    session, chainIndex, userDeviceMap, payload,
+) {
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: this._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };
+    const contentMap = {};
+
+    const promises = [];
+    for (let i = 0; i < userDeviceMap.length; i++) {
+        const val = userDeviceMap[i];
+        const userId = val.userId;
+        const deviceInfo = val.deviceInfo;
+        const deviceId = deviceInfo.deviceId;
+
+        if (!contentMap[userId]) {
+            contentMap[userId] = {};
+        }
+        contentMap[userId][deviceId] = encryptedContent;
+
+        promises.push(
+            olmlib.encryptMessageForDevice(
+                encryptedContent.ciphertext,
+                this._userId,
+                this._deviceId,
+                this._olmDevice,
+                userId,
+                deviceInfo,
+                payload,
+            ),
+        );
+    }
+
+    return Promise.all(promises).then(() => {
+        return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
+            // store that we successfully uploaded the keys of the current slice
+            for (const userId of Object.keys(contentMap)) {
+                for (const deviceId of Object.keys(contentMap[userId])) {
+                    session.markSharedWithDevice(
+                        userId, deviceId, chainIndex,
+                    );
+                }
+            }
+        });
+    });
+};
+
+/**
+ * @private
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices
+ */
+MegolmEncryption.prototype._shareKeyWithDevices = async function(session, devicesByUser) {
    const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
    const payload = {
        type: "m.room_key",
@@ -283,104 +429,28 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
        },
    };

-    const contentMap = {};
-
-    return olmlib.ensureOlmSessionsForDevices(
+    const devicemap = await olmlib.ensureOlmSessionsForDevices(
        this._olmDevice, this._baseApis, devicesByUser,
-    ).then(function(devicemap) {
-        const promises = [];
+    );

-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
+    const userDeviceMaps = this._splitUserDeviceMap(
+        session, key.chain_index, devicemap, devicesByUser,
+    );

-            const devicesToShareWith = devicesByUser[userId];
-            const sessionResults = devicemap[userId];
+    for (let i = 0; i < userDeviceMaps.length; i++) {
+        try {
+            await this._encryptAndSendKeysToDevices(
+                session, key.chain_index, userDeviceMaps[i], payload,
+            );
+            logger.log(`Completed megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length})`);
+        } catch (e) {
+            logger.log(`megolm keyshare in ${this._roomId} `
+                + `(slice ${i + 1}/${userDeviceMaps.length}) failed`);

-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                const deviceId = deviceInfo.deviceId;
-
-                const sessionResult = sessionResults[deviceId];
-                if (!sessionResult.sessionId) {
-                    // no session with this device, probably because there
-                    // were no one-time keys.
-                    //
-                    // we could send them a to_device message anyway, as a
-                    // signal that they have missed out on the key sharing
-                    // message because of the lack of keys, but there's not
-                    // much point in that really; it will mostly serve to clog
-                    // up to_device inboxes.
-                    //
-                    // ensureOlmSessionsForUsers has already done the logging,
-                    // so just skip it.
-                    continue;
-                }
-
-                console.log(
-                    "sharing keys with device " + userId + ":" + deviceId,
-                );
-
-                const encryptedContent = {
-                    algorithm: olmlib.OLM_ALGORITHM,
-                    sender_key: self._olmDevice.deviceCurve25519Key,
-                    ciphertext: {},
-                };
-
-                if (!contentMap[userId]) {
-                    contentMap[userId] = {};
-                }
-
-                contentMap[userId][deviceId] = encryptedContent;
-
-                promises.push(
-                    olmlib.encryptMessageForDevice(
-                        encryptedContent.ciphertext,
-                        self._userId,
-                        self._deviceId,
-                        self._olmDevice,
-                        userId,
-                        deviceInfo,
-                        payload,
-                    ),
-                );
-            }
+            throw e;
        }
-
-        if (promises.length === 0) {
-            // no devices to send to
-            return Promise.resolve();
-        }
-
-        return Promise.all(promises).then(() => {
-            // TODO: retries
-            return self._baseApis.sendToDevice("m.room.encrypted", contentMap);
-        });
-    }).then(function() {
-        console.log(`Completed megolm keyshare in ${self._roomId}`);
-
-        // Add the devices we have shared with to session.sharedWithDevices.
-        //
-        // we deliberately iterate over devicesByUser (ie, the devices we
-        // attempted to share with) rather than the contentMap (those we did
-        // share with), because we don't want to try to claim a one-time-key
-        // for dead devices on every message.
-        for (const userId in devicesByUser) {
-            if (!devicesByUser.hasOwnProperty(userId)) {
-                continue;
-            }
-            if (!session.sharedWithDevices[userId]) {
-                session.sharedWithDevices[userId] = {};
-            }
-            const devicesToShareWith = devicesByUser[userId];
-            for (let i = 0; i < devicesToShareWith.length; i++) {
-                const deviceInfo = devicesToShareWith[i];
-                session.sharedWithDevices[userId][deviceInfo.deviceId] =
-                    key.chain_index;
-            }
-        }
-    });
+    }
 };

 /**
@@ -394,7 +464,7 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUse
 */
 MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
    const self = this;
-    console.log(`Starting to encrypt event for ${this._roomId}`);
+    logger.log(`Starting to encrypt event for ${this._roomId}`);

    return this._getDevicesInRoom(room).then(function(devicesInRoom) {
        // check if any of these devices are not yet known to the user.
@@ -420,6 +490,8 @@ MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
            session_id: session.sessionId,
             // Include our device ID so that recipients can send us a
             // m.new_device message if they don't have our session key.
+             // XXX: Do we still need this now that m.new_device messages
+             // no longer exist since #483?
            device_id: self._deviceId,
        };

@@ -428,6 +500,16 @@ MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
    });
 };

+/**
+ * Forces the current outbound group session to be discarded such
+ * that another one will be created next time an event is sent.
+ *
+ * This should not normally be necessary.
+ */
+MegolmEncryption.prototype.forceDiscardSession = function() {
+    this._setupPromise = this._setupPromise.then(() => null);
+};
+
 /**
 * Checks the devices we're about to send to and see if any are entirely
 * unknown to the user.  If so, warn the user, and mark them as known to
@@ -467,46 +549,46 @@ MegolmEncryption.prototype._checkForUnknownDevices = function(devicesInRoom) {
 * @return {module:client.Promise} Promise which resolves to a map
 *     from userId to deviceId to deviceInfo
 */
-MegolmEncryption.prototype._getDevicesInRoom = function(room) {
-    // XXX what about rooms where invitees can see the content?
-    const roomMembers = utils.map(room.getJoinedMembers(), function(u) {
+MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
+    const members = await room.getEncryptionTargetMembers();
+    const roomMembers = utils.map(members, function(u) {
        return u.userId;
    });

+    // The global value is treated as a default for when rooms don't specify a value.
+    let isBlacklisting = this._crypto.getGlobalBlacklistUnverifiedDevices();
+    if (typeof room.getBlacklistUnverifiedDevices() === 'boolean') {
+        isBlacklisting = room.getBlacklistUnverifiedDevices();
+    }
+
    // We are happy to use a cached version here: we assume that if we already
    // have a list of the user's devices, then we already share an e2e room
    // with them, which means that they will have announced any new devices via
-    // an m.new_device.
-    //
-    // XXX: what if the cache is stale, and the user left the room we had in
-    // common and then added new devices before joining this one? --Matthew
-    //
-    // yup, see https://github.com/vector-im/riot-web/issues/2305 --richvdh
-    return this._crypto.downloadKeys(roomMembers, false).then((devices) => {
-        // remove any blocked devices
-        for (const userId in devices) {
-            if (!devices.hasOwnProperty(userId)) {
+    // device_lists in their /sync response.  This cache should then be maintained
+    // using all the device_lists changes and left fields.
+    // See https://github.com/vector-im/riot-web/issues/2305 for details.
+    const devices = await this._crypto.downloadKeys(roomMembers, false);
+    // remove any blocked devices
+    for (const userId in devices) {
+        if (!devices.hasOwnProperty(userId)) {
+            continue;
+        }
+
+        const userDevices = devices[userId];
+        for (const deviceId in userDevices) {
+            if (!userDevices.hasOwnProperty(deviceId)) {
                continue;
            }

-            const userDevices = devices[userId];
-            for (const deviceId in userDevices) {
-                if (!userDevices.hasOwnProperty(deviceId)) {
-                    continue;
-                }
-
-                if (userDevices[deviceId].isBlocked() ||
-                    (userDevices[deviceId].isUnverified() &&
-                     (room.getBlacklistUnverifiedDevices() ||
-                      this._crypto.getGlobalBlacklistUnverifiedDevices()))
-                   ) {
-                    delete userDevices[deviceId];
-                }
+            if (userDevices[deviceId].isBlocked() ||
+                (userDevices[deviceId].isUnverified() && isBlacklisting)
+               ) {
+                delete userDevices[deviceId];
            }
        }
+    }

-        return devices;
-    });
+    return devices;
 };

 /**
@@ -522,7 +604,7 @@ function MegolmDecryption(params) {
    base.DecryptionAlgorithm.call(this, params);

    // events which we couldn't decrypt due to unknown sessions / indexes: map from
-    // senderKey|sessionId to list of MatrixEvents
+    // senderKey|sessionId to Set of MatrixEvents
    this._pendingEvents = {};

    // this gets stubbed out by the unit tests.
@@ -546,20 +628,36 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
    if (!content.sender_key || !content.session_id ||
        !content.ciphertext
       ) {
-        throw new base.DecryptionError("Missing fields in input");
+        throw new base.DecryptionError(
+            "MEGOLM_MISSING_FIELDS",
+            "Missing fields in input",
+        );
    }

+    // we add the event to the pending list *before* we start decryption.
+    //
+    // then, if the key turns up while decryption is in progress (and
+    // decryption fails), we will schedule a retry.
+    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    this._addEventToPendingList(event);
+
    let res;
    try {
        res = await this._olmDevice.decryptGroupMessage(
            event.getRoomId(), content.sender_key, content.session_id, content.ciphertext,
+            event.getId(), event.getTs(),
        );
    } catch (e) {
+        let errorCode = "OLM_DECRYPT_GROUP_MESSAGE_ERROR";
+
        if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
-            this._addEventToPendingList(event);
            this._requestKeysForEvent(event);
+
+            errorCode = 'OLM_UNKNOWN_MESSAGE_INDEX';
        }
+
        throw new base.DecryptionError(
+            errorCode,
            e.toString(), {
                session: content.sender_key + '|' + content.session_id,
            },
@@ -568,9 +666,15 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {

    if (res === null) {
        // We've got a message for a session we don't have.
-        this._addEventToPendingList(event);
+        //
+        // (XXX: We might actually have received this key since we started
+        // decrypting, in which case we'll have scheduled a retry, and this
+        // request will be redundant. We could probably check to see if the
+        // event is still in the pending list; if not, a retry will have been
+        // scheduled, so we needn't send out the request here.)
        this._requestKeysForEvent(event);
        throw new base.DecryptionError(
+            "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
            "The sender's device has not sent us the keys for this message.",
            {
                session: content.sender_key + '|' + content.session_id,
@@ -578,6 +682,10 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
        );
    }

+    // success. We can remove the event from the pending list, if that hasn't
+    // already happened.
+    this._removeEventFromPendingList(event);
+
    const payload = JSON.parse(res.result);

    // belt-and-braces check that the room id matches that indicated by the HS
@@ -585,6 +693,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
    // room, so neither the sender nor a MITM can lie about the room_id).
    if (payload.room_id !== event.getRoomId()) {
        throw new base.DecryptionError(
+            "MEGOLM_BAD_ROOM",
            "Message intended for room " + payload.room_id,
        );
    }
@@ -621,8 +730,7 @@ MegolmDecryption.prototype._requestKeysForEvent = function(event) {
 };

 /**
- * Add an event to the list of those we couldn't decrypt the first time we
- * saw them.
+ * Add an event to the list of those awaiting their session keys.
 *
 * @private
 *
@@ -632,11 +740,32 @@ MegolmDecryption.prototype._addEventToPendingList = function(event) {
    const content = event.getWireContent();
    const k = content.sender_key + "|" + content.session_id;
    if (!this._pendingEvents[k]) {
-        this._pendingEvents[k] = [];
+        this._pendingEvents[k] = new Set();
    }
-    this._pendingEvents[k].push(event);
+    this._pendingEvents[k].add(event);
 };

+/**
+ * Remove an event from the list of those awaiting their session keys.
+ *
+ * @private
+ *
+ * @param {module:models/event.MatrixEvent} event
+ */
+MegolmDecryption.prototype._removeEventFromPendingList = function(event) {
+    const content = event.getWireContent();
+    const k = content.sender_key + "|" + content.session_id;
+    if (!this._pendingEvents[k]) {
+        return;
+    }
+
+    this._pendingEvents[k].delete(event);
+    if (this._pendingEvents[k].size === 0) {
+        delete this._pendingEvents[k];
+    }
+};
+
+
 /**
 * @inheritdoc
 *
@@ -654,12 +783,12 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        !sessionId ||
        !content.session_key
       ) {
-        console.error("key event is missing fields");
+        logger.error("key event is missing fields");
        return;
    }

    if (!senderKey) {
-        console.error("key event has no sender key (not encrypted?)");
+        logger.error("key event has no sender key (not encrypted?)");
        return;
    }

@@ -676,13 +805,13 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {

        senderKey = content.sender_key;
        if (!senderKey) {
-            console.error("forwarded_room_key event is missing sender_key field");
+            logger.error("forwarded_room_key event is missing sender_key field");
            return;
        }

        const ed25519Key = content.sender_claimed_ed25519_key;
        if (!ed25519Key) {
-            console.error(
+            logger.error(
                `forwarded_room_key_event is missing sender_claimed_ed25519_key field`,
            );
            return;
@@ -695,7 +824,7 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        keysClaimed = event.getKeysClaimed();
    }

-    console.log(`Adding key for megolm session ${senderKey}|${sessionId}`);
+    logger.log(`Adding key for megolm session ${senderKey}|${sessionId}`);
    this._olmDevice.addInboundGroupSession(
        content.room_id, senderKey, forwardingKeyChain, sessionId,
        content.session_key, keysClaimed,
@@ -712,7 +841,7 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        // have another go at decrypting events sent with this session.
        this._retryDecryption(senderKey, sessionId);
    }).catch((e) => {
-        console.error(`Error handling m.room_key_event: ${e}`);
+        logger.error(`Error handling m.room_key_event: ${e}`);
    });
 };

@@ -754,7 +883,7 @@ MegolmDecryption.prototype.shareKeysWithDevice = function(keyRequest) {
            return null;
        }

-        console.log(
+        logger.log(
            "sharing keys for session " + body.sender_key + "|"
            + body.session_id + " with device "
            + userId + ":" + deviceId,
@@ -819,10 +948,18 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
 * @param {module:crypto/OlmDevice.MegolmSessionData} session
 */
 MegolmDecryption.prototype.importRoomKey = function(session) {
-    this._olmDevice.importInboundGroupSession(session);
-
-    // have another go at decrypting events sent with this session.
-    this._retryDecryption(session.sender_key, session.session_id);
+    return this._olmDevice.addInboundGroupSession(
+        session.room_id,
+        session.sender_key,
+        session.forwarding_curve25519_key_chain,
+        session.session_id,
+        session.session_key,
+        session.sender_claimed_keys,
+        true,
+    ).then(() => {
+        // have another go at decrypting events sent with this session.
+        this._retryDecryption(session.sender_key, session.session_id);
+    });
 };

 /**
@@ -841,8 +978,8 @@ MegolmDecryption.prototype._retryDecryption = function(senderKey, sessionId) {

    delete this._pendingEvents[k];

-    for (let i = 0; i < pending.length; i++) {
-        pending[i].attemptDecryption(this._crypto);
+    for (const ev of pending) {
+        ev.attemptDecryption(this._crypto);
    }
 };

@@ -22,6 +22,7 @@ limitations under the License.
 */
 import Promise from 'bluebird';

+const logger = require("../../logger");
 const utils = require("../../utils");
 const olmlib = require("../olmlib");
 const DeviceInfo = require("../deviceinfo");
@@ -83,60 +84,62 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
 *
 * @return {module:client.Promise} Promise which resolves to the new event body
 */
-OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {
+OlmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
    // pick the list of recipients based on the membership list.
    //
    // TODO: there is a race condition here! What if a new user turns up
    // just as you are sending a secret message?

-    const users = utils.map(room.getJoinedMembers(), function(u) {
+    const members = await room.getEncryptionTargetMembers();
+
+    const users = utils.map(members, function(u) {
        return u.userId;
    });

    const self = this;
-    return this._ensureSession(users).then(function() {
-        const payloadFields = {
-            room_id: room.roomId,
-            type: eventType,
-            content: content,
-        };
+    await this._ensureSession(users);

-        const encryptedContent = {
-            algorithm: olmlib.OLM_ALGORITHM,
-            sender_key: self._olmDevice.deviceCurve25519Key,
-            ciphertext: {},
-        };
+    const payloadFields = {
+        room_id: room.roomId,
+        type: eventType,
+        content: content,
+    };

-        const promises = [];
+    const encryptedContent = {
+        algorithm: olmlib.OLM_ALGORITHM,
+        sender_key: self._olmDevice.deviceCurve25519Key,
+        ciphertext: {},
+    };

-        for (let i = 0; i < users.length; ++i) {
-            const userId = users[i];
-            const devices = self._crypto.getStoredDevicesForUser(userId);
+    const promises = [];

-            for (let j = 0; j < devices.length; ++j) {
-                const deviceInfo = devices[j];
-                const key = deviceInfo.getIdentityKey();
-                if (key == self._olmDevice.deviceCurve25519Key) {
-                    // don't bother sending to ourself
-                    continue;
-                }
-                if (deviceInfo.verified == DeviceVerification.BLOCKED) {
-                    // don't bother setting up sessions with blocked users
-                    continue;
-                }
+    for (let i = 0; i < users.length; ++i) {
+        const userId = users[i];
+        const devices = self._crypto.getStoredDevicesForUser(userId);

-                promises.push(
-                    olmlib.encryptMessageForDevice(
-                        encryptedContent.ciphertext,
-                        self._userId, self._deviceId, self._olmDevice,
-                        userId, deviceInfo, payloadFields,
-                    ),
-                );
+        for (let j = 0; j < devices.length; ++j) {
+            const deviceInfo = devices[j];
+            const key = deviceInfo.getIdentityKey();
+            if (key == self._olmDevice.deviceCurve25519Key) {
+                // don't bother sending to ourself
+                continue;
+            }
+            if (deviceInfo.verified == DeviceVerification.BLOCKED) {
+                // don't bother setting up sessions with blocked users
+                continue;
            }
-        }

-        return Promise.all(promises).return(encryptedContent);
-    });
+            promises.push(
+                olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    self._userId, self._deviceId, self._olmDevice,
+                    userId, deviceInfo, payloadFields,
+                ),
+            );
+        }
+    }
+
+    return await Promise.all(promises).return(encryptedContent);
 };

 /**
@@ -168,11 +171,17 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
    const ciphertext = content.ciphertext;

    if (!ciphertext) {
-        throw new base.DecryptionError("Missing ciphertext");
+        throw new base.DecryptionError(
+            "OLM_MISSING_CIPHERTEXT",
+            "Missing ciphertext",
+        );
    }

    if (!(this._olmDevice.deviceCurve25519Key in ciphertext)) {
-        throw new base.DecryptionError("Not included in recipients");
+        throw new base.DecryptionError(
+            "OLM_NOT_INCLUDED_IN_RECIPIENTS",
+            "Not included in recipients",
+        );
    }
    const message = ciphertext[this._olmDevice.deviceCurve25519Key];
    let payloadString;
@@ -181,6 +190,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
        payloadString = await this._decryptMessage(deviceKey, message);
    } catch (e) {
        throw new base.DecryptionError(
+            "OLM_BAD_ENCRYPTED_MESSAGE",
            "Bad Encrypted Message", {
                sender: deviceKey,
                err: e,
@@ -194,12 +204,14 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
    // https://github.com/vector-im/vector-web/issues/2483
    if (payload.recipient != this._userId) {
        throw new base.DecryptionError(
+            "OLM_BAD_RECIPIENT",
            "Message was intented for " + payload.recipient,
        );
    }

    if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
        throw new base.DecryptionError(
+            "OLM_BAD_RECIPIENT_KEY",
            "Message not intended for this device", {
                intended: payload.recipient_keys.ed25519,
                our_key: this._olmDevice.deviceEd25519Key,
@@ -213,6 +225,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
    // which is checked elsewhere).
    if (payload.sender != event.getSender()) {
        throw new base.DecryptionError(
+            "OLM_FORWARDED_MESSAGE",
            "Message forwarded from " + payload.sender, {
                reported_sender: event.getSender(),
            },
@@ -222,6 +235,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
    // Olm events intended for a room have a room_id.
    if (payload.room_id !== event.getRoomId()) {
        throw new base.DecryptionError(
+            "OLM_BAD_ROOM",
            "Message intended for room " + payload.room_id, {
                reported_room: event.room_id,
            },
@@ -260,7 +274,7 @@ OlmDecryption.prototype._decryptMessage = async function(
            const payload = await this._olmDevice.decryptMessage(
                theirDeviceIdentityKey, sessionId, message.type, message.body,
            );
-            console.log(
+            logger.log(
                "Decrypted Olm message from " + theirDeviceIdentityKey +
                    " with session " + sessionId,
            );
@@ -315,7 +329,7 @@ OlmDecryption.prototype._decryptMessage = async function(
        );
    }

-    console.log(
+    logger.log(
        "created new inbound Olm session ID " +
            res.session_id + " with " + theirDeviceIdentityKey,
    );
@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,6 +25,7 @@ const anotherjson = require('another-json');
 import Promise from 'bluebird';
 import {EventEmitter} from 'events';

+const logger = require("../logger");
 const utils = require("../utils");
 const OlmDevice = require("./OlmDevice");
 const olmlib = require("./olmlib");
@@ -33,6 +35,11 @@ const DeviceVerification = DeviceInfo.DeviceVerification;
 const DeviceList = require('./DeviceList').default;

 import OutgoingRoomKeyRequestManager from './OutgoingRoomKeyRequestManager';
+import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+
+export function isCryptoAvailable() {
+    return Boolean(global.Olm);
+}

 /**
 * Cryptography bits
@@ -57,18 +64,23 @@ import OutgoingRoomKeyRequestManager from './OutgoingRoomKeyRequestManager';
 *
 * @param {module:crypto/store/base~CryptoStore} cryptoStore
 *    storage for the crypto layer.
+ *
+ * @param {RoomList} roomList An initialised RoomList object
 */
-function Crypto(baseApis, sessionStore, userId, deviceId,
-                clientStore, cryptoStore) {
+export default function Crypto(baseApis, sessionStore, userId, deviceId,
+                clientStore, cryptoStore, roomList) {
    this._baseApis = baseApis;
    this._sessionStore = sessionStore;
    this._userId = userId;
    this._deviceId = deviceId;
    this._clientStore = clientStore;
    this._cryptoStore = cryptoStore;
+    this._roomList = roomList;

-    this._olmDevice = new OlmDevice(sessionStore);
-    this._deviceList = new DeviceList(baseApis, sessionStore, this._olmDevice);
+    this._olmDevice = new OlmDevice(sessionStore, cryptoStore);
+    this._deviceList = new DeviceList(
+        baseApis, cryptoStore, sessionStore, this._olmDevice,
+    );

    // the last time we did a check for the number of one-time-keys on the
    // server.
@@ -99,6 +111,15 @@ function Crypto(baseApis, sessionStore, userId, deviceId,
    this._receivedRoomKeyRequestCancellations = [];
    // true if we are currently processing received room key requests
    this._processingRoomKeyRequests = false;
+    // controls whether device tracking is delayed
+    // until calling encryptEvent or trackRoomDevices,
+    // or done immediately upon enabling room encryption.
+    this._lazyLoadMembers = false;
+    // in case _lazyLoadMembers is true,
+    // track if an initial tracking of all the room members
+    // has happened for a given room. This is delayed
+    // to avoid loading room members as long as possible.
+    this._roomDeviceTrackingState = {};
 }
 utils.inherits(Crypto, EventEmitter);

@@ -108,7 +129,28 @@ utils.inherits(Crypto, EventEmitter);
 * Returns a promise which resolves once the crypto module is ready for use.
 */
 Crypto.prototype.init = async function() {
+    await global.Olm.init();
+
+    const sessionStoreHasAccount = Boolean(this._sessionStore.getEndToEndAccount());
+    let cryptoStoreHasAccount;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
+            this._cryptoStore.getAccount(txn, (pickledAccount) => {
+                cryptoStoreHasAccount = Boolean(pickledAccount);
+            });
+        },
+    );
+    if (sessionStoreHasAccount && !cryptoStoreHasAccount) {
+        // we're about to migrate to the crypto store
+        this.emit("crypto.warning", 'CRYPTO_WARNING_ACCOUNT_MIGRATED');
+    } else if (sessionStoreHasAccount && cryptoStoreHasAccount) {
+        // There's an account in both stores: an old version of
+        // the code has been run against this store.
+        this.emit("crypto.warning", 'CRYPTO_WARNING_OLD_VERSION_DETECTED');
+    }
+
    await this._olmDevice.init();
+    await this._deviceList.load();

    // build our device keys: these will later be uploaded
    this._deviceKeys["ed25519:" + this._deviceId] =
@@ -116,7 +158,7 @@ Crypto.prototype.init = async function() {
    this._deviceKeys["curve25519:" + this._deviceId] =
        this._olmDevice.deviceCurve25519Key;

-    let myDevices = this._sessionStore.getEndToEndDevicesForUser(
+    let myDevices = this._deviceList.getRawStoredDevicesForUser(
        this._userId,
    );

@@ -134,12 +176,19 @@ Crypto.prototype.init = async function() {
        };

        myDevices[this._deviceId] = deviceInfo;
-        this._sessionStore.storeEndToEndDevicesForUser(
+        this._deviceList.storeDevicesForUser(
            this._userId, myDevices,
        );
+        this._deviceList.saveIfDirty();
    }
 };

+/**
+ */
+Crypto.prototype.enableLazyLoading = function() {
+    this._lazyLoadMembers = true;
+};
+
 /**
 * Tell the crypto module to register for MatrixClient events which it needs to
 * listen for
@@ -154,7 +203,7 @@ Crypto.prototype.registerEventHandlers = function(eventEmitter) {
        try {
            crypto._onRoomMembership(event, member, oldMembership);
        } catch (e) {
-             console.error("Error handling membership change:", e);
+             logger.error("Error handling membership change:", e);
        }
    });

@@ -172,6 +221,7 @@ Crypto.prototype.start = function() {
 /** Stop background processes related to crypto */
 Crypto.prototype.stop = function() {
    this._outgoingRoomKeyRequestManager.stop();
+    this._deviceList.stop();
 };

 /**
@@ -192,18 +242,17 @@ Crypto.prototype.getDeviceEd25519Key = function() {

 /**
 * Set the global override for whether the client should ever send encrypted
- * messages to unverified devices.  If false, it can still be overridden
- * per-room.  If true, it overrides the per-room settings.
+ * messages to unverified devices.  This provides the default for rooms which
+ * do not specify a value.
 *
- * @param {boolean} value whether to unilaterally blacklist all
- * unverified devices
+ * @param {boolean} value whether to blacklist all unverified devices by default
 */
 Crypto.prototype.setGlobalBlacklistUnverifiedDevices = function(value) {
    this._globalBlacklistUnverifiedDevices = value;
 };

 /**
- * @return {boolean} whether to unilaterally blacklist all unverified devices
+ * @return {boolean} whether to blacklist all unverified devices by default
 */
 Crypto.prototype.getGlobalBlacklistUnverifiedDevices = function() {
    return this._globalBlacklistUnverifiedDevices;
@@ -340,7 +389,7 @@ function _maybeUploadOneTimeKeys(crypto) {
        // create any more keys.
        return uploadLoop(keyCount);
    }).catch((e) => {
-        console.error("Error uploading one-time keys", e.stack || e);
+        logger.error("Error uploading one-time keys", e.stack || e);
    }).finally(() => {
        // reset _oneTimeKeyCount to prevent start uploading based on old data.
        // it will be set again on the next /sync-response
@@ -418,6 +467,22 @@ Crypto.prototype.getStoredDevice = function(userId, deviceId) {
    return this._deviceList.getStoredDevice(userId, deviceId);
 };

+/**
+ * Save the device list, if necessary
+ *
+ * @param {integer} delay Time in ms before which the save actually happens.
+ *     By default, the save is delayed for a short period in order to batch
+ *     multiple writes, but this behaviour can be disabled by passing 0.
+ *
+ * @return {Promise<bool>} true if the data was saved, false if
+ *     it was not (eg. because no changes were pending). The promise
+ *     will only resolve once the data is saved, so may take some time
+ *     to resolve.
+ */
+Crypto.prototype.saveDeviceList = function(delay) {
+    return this._deviceList.saveIfDirty(delay);
+};
+
 /**
 * Update the blocked/verified state of the given device
 *
@@ -438,7 +503,7 @@ Crypto.prototype.getStoredDevice = function(userId, deviceId) {
 Crypto.prototype.setDeviceVerification = async function(
    userId, deviceId, verified, blocked, known,
 ) {
-    const devices = this._sessionStore.getEndToEndDevicesForUser(userId);
+    const devices = this._deviceList.getRawStoredDevicesForUser(userId);
    if (!devices || !devices[deviceId]) {
        throw new Error("Unknown device " + userId + ":" + deviceId);
    }
@@ -466,7 +531,8 @@ Crypto.prototype.setDeviceVerification = async function(
    if (dev.verified !== verificationStatus || dev.known !== knownStatus) {
        dev.verified = verificationStatus;
        dev.known = knownStatus;
-        this._sessionStore.storeEndToEndDevicesForUser(userId, devices);
+        this._deviceList.storeDevicesForUser(userId, devices);
+        this._deviceList.saveIfDirty();
    }
    return DeviceInfo.fromStorage(dev, deviceId);
 };
@@ -548,13 +614,13 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {

    const claimedKey = event.getClaimedEd25519Key();
    if (!claimedKey) {
-        console.warn("Event " + event.getId() + " claims no ed25519 key: " +
+        logger.warn("Event " + event.getId() + " claims no ed25519 key: " +
                     "cannot verify sending device");
        return null;
    }

    if (claimedKey !== device.getFingerprint()) {
-        console.warn(
+        logger.warn(
            "Event " + event.getId() + " claims ed25519 key " + claimedKey +
                "but sender device has key " + device.getFingerprint());
        return null;
@@ -563,6 +629,22 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
    return device;
 };

+/**
+ * Forces the current outbound group session to be discarded such
+ * that another one will be created next time an event is sent.
+ *
+ * @param {string} roomId The ID of the room to discard the session for
+ *
+ * This should not normally be necessary.
+ */
+Crypto.prototype.forceDiscardSession = function(roomId) {
+    const alg = this._roomEncryptors[roomId];
+    if (alg === undefined) throw new Error("Room not encrypted");
+    if (alg.forceDiscardSession === undefined) {
+        throw new Error("Room encryption algorithm doesn't support session discarding");
+    }
+    alg.forceDiscardSession();
+};

 /**
 * Configure a room to use encryption (ie, save a flag in the sessionstore).
@@ -572,27 +654,49 @@ Crypto.prototype.getEventSenderDeviceInfo = function(event) {
 * @param {object} config The encryption config for the room.
 *
 * @param {boolean=} inhibitDeviceQuery true to suppress device list query for
- *   users in the room (for now)
+ *   users in the room (for now). In case lazy loading is enabled,
+ *   the device query is always inhibited as the members are not tracked.
 */
 Crypto.prototype.setRoomEncryption = async function(roomId, config, inhibitDeviceQuery) {
-    // if we already have encryption in this room, we should ignore this event
-    // (for now at least. maybe we should alert the user somehow?)
-    const existingConfig = this._sessionStore.getEndToEndRoom(roomId);
+    // if state is being replayed from storage, we might already have a configuration
+    // for this room as they are persisted as well.
+    // We just need to make sure the algorithm is initialized in this case.
+    // However, if the new config is different,
+    // we should bail out as room encryption can't be changed once set.
+    const existingConfig = this._roomList.getRoomEncryption(roomId);
    if (existingConfig) {
        if (JSON.stringify(existingConfig) != JSON.stringify(config)) {
-            console.error("Ignoring m.room.encryption event which requests " +
+            logger.error("Ignoring m.room.encryption event which requests " +
                          "a change of config in " + roomId);
            return;
        }
    }
+    // if we already have encryption in this room, we should ignore this event,
+    // as it would reset the encryption algorithm.
+    // This is at least expected to be called twice, as sync calls onCryptoEvent
+    // for both the timeline and state sections in the /sync response,
+    // the encryption event would appear in both.
+    // If it's called more than twice though,
+    // it signals a bug on client or server.
+    const existingAlg = this._roomEncryptors[roomId];
+    if (existingAlg) {
+        return;
+    }
+
+    // _roomList.getRoomEncryption will not race with _roomList.setRoomEncryption
+    // because it first stores in memory. We should await the promise only
+    // after all the in-memory state (_roomEncryptors and _roomList) has been updated
+    // to avoid races when calling this method multiple times. Hence keep a hold of the promise.
+    let storeConfigPromise = null;
+    if(!existingConfig) {
+        storeConfigPromise = this._roomList.setRoomEncryption(roomId, config);
+    }

    const AlgClass = algorithms.ENCRYPTION_CLASSES[config.algorithm];
    if (!AlgClass) {
        throw new Error("Unable to encrypt with " + config.algorithm);
    }

-    this._sessionStore.storeEndToEndRoom(roomId, config);
-
    const alg = new AlgClass({
        userId: this._userId,
        deviceId: this._deviceId,
@@ -604,24 +708,59 @@ Crypto.prototype.setRoomEncryption = async function(roomId, config, inhibitDevic
    });
    this._roomEncryptors[roomId] = alg;

-    // make sure we are tracking the device lists for all users in this room.
-    console.log("Enabling encryption in " + roomId + "; " +
-                "starting to track device lists for all users therein");
-    const room = this._clientStore.getRoom(roomId);
-    if (!room) {
-        throw new Error(`Unable to enable encryption in unknown room ${roomId}`);
+    if (storeConfigPromise) {
+        await storeConfigPromise;
    }

-    const members = room.getJoinedMembers();
-    members.forEach((m) => {
-        this._deviceList.startTrackingDeviceList(m.userId);
-    });
-    if (!inhibitDeviceQuery) {
-        this._deviceList.refreshOutdatedDeviceLists();
+    if (!this._lazyLoadMembers) {
+        logger.log("Enabling encryption in " + roomId + "; " +
+            "starting to track device lists for all users therein");
+
+        await this.trackRoomDevices(roomId);
+        // TODO: this flag is only not used from MatrixClient::setRoomEncryption
+        // which is never used (inside riot at least)
+        // but didn't want to remove it as it technically would
+        // be a breaking change.
+        if(!this.inhibitDeviceQuery) {
+            this._deviceList.refreshOutdatedDeviceLists();
+        }
+    } else {
+        logger.log("Enabling encryption in " + roomId);
    }
 };


+/**
+ * Make sure we are tracking the device lists for all users in this room.
+ *
+ * @param {string} roomId The room ID to start tracking devices in.
+ * @returns {Promise} when all devices for the room have been fetched and marked to track
+ */
+Crypto.prototype.trackRoomDevices = function(roomId) {
+    const trackMembers = async () => {
+        // not an encrypted room
+        if (!this._roomEncryptors[roomId]) {
+            return;
+        }
+        const room = this._clientStore.getRoom(roomId);
+        if (!room) {
+            throw new Error(`Unable to start tracking devices in unknown room ${roomId}`);
+        }
+        logger.log(`Starting to track devices for room ${roomId} ...`);
+        const members = await room.getEncryptionTargetMembers();
+        members.forEach((m) => {
+            this._deviceList.startTrackingDeviceList(m.userId);
+        });
+    };
+
+    let promise = this._roomDeviceTrackingState[roomId];
+    if (!promise) {
+        promise = trackMembers();
+        this._roomDeviceTrackingState[roomId] = promise;
+    }
+    return promise;
+};
+
 /**
 * @typedef {Object} module:crypto~OlmSessionResult
 * @property {module:crypto/deviceinfo} device  device info
@@ -669,34 +808,28 @@ Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
    );
 };

-/**
- * Whether encryption is enabled for a room.
- * @param {string} roomId the room id to query.
- * @return {bool} whether encryption is enabled.
- */
-Crypto.prototype.isRoomEncrypted = function(roomId) {
-    return Boolean(this._roomEncryptors[roomId]);
-};
-
-
 /**
 * Get a list containing all of the room keys
 *
- * @return {module:client.Promise} a promise which resolves to a list of
- *    session export objects
+ * @return {module:crypto/OlmDevice.MegolmSessionData[]} a list of session export objects
 */
-Crypto.prototype.exportRoomKeys = function() {
-    return Promise.map(
-        this._sessionStore.getAllEndToEndInboundGroupSessionKeys(),
-        (s) => {
-            return this._olmDevice.exportInboundGroupSession(
-                s.senderKey, s.sessionId,
-            ).then((sess) => {
+Crypto.prototype.exportRoomKeys = async function() {
+    const exportedSessions = [];
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+            this._cryptoStore.getAllEndToEndInboundGroupSessions(txn, (s) => {
+                if (s === null) return;
+
+                const sess = this._olmDevice.exportInboundGroupSession(
+                    s.senderKey, s.sessionId, s.sessionData,
+                );
                sess.algorithm = olmlib.MEGOLM_ALGORITHM;
-                return sess;
+                exportedSessions.push(sess);
            });
        },
    );
+
+    return exportedSessions;
 };

 /**
@@ -709,7 +842,7 @@ Crypto.prototype.importRoomKeys = function(keys) {
    return Promise.map(
        keys, (key) => {
            if (!key.room_id || !key.algorithm) {
-                console.warn("ignoring room key entry with missing fields", key);
+                logger.warn("ignoring room key entry with missing fields", key);
                return null;
            }

@@ -718,7 +851,7 @@ Crypto.prototype.importRoomKeys = function(keys) {
        },
    );
 };
-
+/* eslint-disable valid-jsdoc */    //https://github.com/eslint/eslint/issues/7307
 /**
 * Encrypt an event according to the configuration of the room.
 *
@@ -729,7 +862,8 @@ Crypto.prototype.importRoomKeys = function(keys) {
 * @return {module:client.Promise?} Promise which resolves when the event has been
 *     encrypted, or null if nothing was needed
 */
-Crypto.prototype.encryptEvent = function(event, room) {
+/* eslint-enable valid-jsdoc */
+Crypto.prototype.encryptEvent = async function(event, room) {
    if (!room) {
        throw new Error("Cannot send encrypted messages in unknown rooms");
    }
@@ -747,16 +881,35 @@ Crypto.prototype.encryptEvent = function(event, room) {
        );
    }

-    return alg.encryptMessage(
-        room, event.getType(), event.getContent(),
-    ).then((encryptedContent) => {
-        event.makeEncrypted(
-            "m.room.encrypted",
-            encryptedContent,
-            this._olmDevice.deviceCurve25519Key,
-            this._olmDevice.deviceEd25519Key,
-        );
-    });
+    if (!this._roomDeviceTrackingState[roomId]) {
+        this.trackRoomDevices(roomId);
+    }
+    // wait for all the room devices to be loaded
+    await this._roomDeviceTrackingState[roomId];
+
+    let content = event.getContent();
+    // If event has an m.relates_to then we need
+    // to put this on the wrapping event instead
+    const mRelatesTo = content['m.relates_to'];
+    if (mRelatesTo) {
+        // Clone content here so we don't remove `m.relates_to` from the local-echo
+        content = Object.assign({}, content);
+        delete content['m.relates_to'];
+    }
+
+    const encryptedContent = await alg.encryptMessage(
+        room, event.getType(), content);
+
+    if (mRelatesTo) {
+        encryptedContent['m.relates_to'] = mRelatesTo;
+    }
+
+    event.makeEncrypted(
+        "m.room.encrypted",
+        encryptedContent,
+        this._olmDevice.deviceCurve25519Key,
+        this._olmDevice.deviceEd25519Key,
+    );
 };

 /**
@@ -769,6 +922,15 @@ Crypto.prototype.encryptEvent = function(event, room) {
 *  is a problem decrypting the event.
 */
 Crypto.prototype.decryptEvent = function(event) {
+    if (event.isRedacted()) {
+        return Promise.resolve({
+            clearEvent: {
+                room_id: event.getRoomId(),
+                type: "m.room.message",
+                content: {},
+            },
+        });
+    }
    const content = event.getWireContent();
    const alg = this._getRoomDecryptor(event.getRoomId(), content.algorithm);
    return alg.decryptEvent(event);
@@ -778,24 +940,24 @@ Crypto.prototype.decryptEvent = function(event) {
 * Handle the notification from /sync or /keys/changes that device lists have
 * been changed.
 *
- * @param {Object} deviceLists device_lists field from /sync, or response from
+ * @param {Object} syncData Object containing sync tokens associated with this sync
+ * @param {Object} syncDeviceLists device_lists field from /sync, or response from
 * /keys/changes
 */
-Crypto.prototype.handleDeviceListChanges = async function(deviceLists) {
-    if (deviceLists.changed && Array.isArray(deviceLists.changed)) {
-        deviceLists.changed.forEach((u) => {
-            this._deviceList.invalidateUserDeviceList(u);
-        });
-    }
+Crypto.prototype.handleDeviceListChanges = async function(syncData, syncDeviceLists) {
+    // Initial syncs don't have device change lists. We'll either get the complete list
+    // of changes for the interval or will have invalidated everything in willProcessSync
+    if (!syncData.oldSyncToken) return;

-    if (deviceLists.left && Array.isArray(deviceLists.left)) {
-        deviceLists.left.forEach((u) => {
-            this._deviceList.stopTrackingDeviceList(u);
-        });
-    }
-
-    // don't flush the outdated device list yet - we do it once we finish
-    // processing the sync.
+    // Here, we're relying on the fact that we only ever save the sync data after
+    // sucessfully saving the device list data, so we're guaranteed that the device
+    // list store is at least as fresh as the sync token from the sync store, ie.
+    // any device changes received in sync tokens prior to the 'next' token here
+    // have been processed and are reflected in the current device list.
+    // If we didn't make this assumption, we'd have to use the /keys/changes API
+    // to get key changes between the sync token in the device list and the 'old'
+    // sync token used here to make sure we didn't miss any.
+    await this._evalDeviceListChanges(syncDeviceLists);
 };

 /**
@@ -809,7 +971,7 @@ Crypto.prototype.requestRoomKey = function(requestBody, recipients) {
        requestBody, recipients,
    ).catch((e) => {
        // this normally means we couldn't talk to the store
-        console.error(
+        logger.error(
            'Error requesting key for event', e,
        );
    }).done();
@@ -820,11 +982,13 @@ Crypto.prototype.requestRoomKey = function(requestBody, recipients) {
 *
 * @param {module:crypto~RoomKeyRequestBody} requestBody
 *    parameters to match for cancellation
+ * @param {boolean} andResend
+ *    if true, resend the key request after cancelling.
 */
-Crypto.prototype.cancelRoomKeyRequest = function(requestBody) {
-    this._outgoingRoomKeyRequestManager.cancelRoomKeyRequest(requestBody)
+Crypto.prototype.cancelRoomKeyRequest = function(requestBody, andResend) {
+    this._outgoingRoomKeyRequestManager.cancelRoomKeyRequest(requestBody, andResend)
    .catch((e) => {
-        console.warn("Error clearing pending room key requests", e);
+        logger.warn("Error clearing pending room key requests", e);
    }).done();
 };

@@ -842,11 +1006,28 @@ Crypto.prototype.onCryptoEvent = async function(event) {
        // finished processing the sync, in onSyncCompleted.
        await this.setRoomEncryption(roomId, content, true);
    } catch (e) {
-        console.error("Error configuring encryption in room " + roomId +
+        logger.error("Error configuring encryption in room " + roomId +
                      ":", e);
    }
 };

+/**
+ * Called before the result of a sync is procesed
+ *
+ * @param {Object} syncData  the data from the 'MatrixClient.sync' event
+ */
+Crypto.prototype.onSyncWillProcess = async function(syncData) {
+    if (!syncData.oldSyncToken) {
+        // If there is no old sync token, we start all our tracking from
+        // scratch, so mark everything as untracked. onCryptoEvent will
+        // be called for all e2e rooms during the processing of the sync,
+        // at which point we'll start tracking all the users of that room.
+        logger.log("Initial sync performed - resetting device tracking state");
+        this._deviceList.stopTrackingAllDeviceLists();
+        this._roomDeviceTrackingState = {};
+    }
+};
+
 /**
 * handle the completion of a /sync
 *
@@ -858,37 +1039,8 @@ Crypto.prototype.onCryptoEvent = async function(event) {
 Crypto.prototype.onSyncCompleted = async function(syncData) {
    const nextSyncToken = syncData.nextSyncToken;

-    if (!syncData.oldSyncToken) {
-        console.log("Completed initial sync");
-
-        // if we have a deviceSyncToken, we can tell the deviceList to
-        // invalidate devices which have changed since then.
-        const oldSyncToken = this._sessionStore.getEndToEndDeviceSyncToken();
-        if (oldSyncToken !== null) {
-            try {
-                await this._invalidateDeviceListsSince(
-                    oldSyncToken, nextSyncToken,
-                );
-            } catch (e) {
-                // if that failed, we fall back to invalidating everyone.
-                console.warn("Error fetching changed device list", e);
-                this._deviceList.invalidateAllDeviceLists();
-            }
-        } else {
-            // otherwise, we have to invalidate all devices for all users we
-            // are tracking.
-            console.log("Completed first initialsync; invalidating all " +
-                        "device list caches");
-            this._deviceList.invalidateAllDeviceLists();
-        }
-    }
-
-    // we can now store our sync token so that we can get an update on
-    // restart rather than having to invalidate everyone.
-    //
-    // (we don't really need to do this on every sync - we could just
-    // do it periodically)
-    this._sessionStore.storeEndToEndDeviceSyncToken(nextSyncToken);
+    this._deviceList.setSyncToken(syncData.nextSyncToken);
+    this._deviceList.saveIfDirty();

    // catch up on any new devices we got told about during the sync.
    this._deviceList.lastKnownSyncToken = nextSyncToken;
@@ -905,49 +1057,71 @@ Crypto.prototype.onSyncCompleted = async function(syncData) {
 };

 /**
- * Ask the server which users have new devices since a given token,
- * and invalidate them
+ * Trigger the appropriate invalidations and removes for a given
+ * device list
 *
- * @param {String} oldSyncToken
- * @param {String} lastKnownSyncToken
- *
- * Returns a Promise which resolves once the query is complete. Rejects if the
- *   keyChange query fails.
+ * @param {Object} deviceLists device_lists field from /sync, or response from
+ * /keys/changes
 */
-Crypto.prototype._invalidateDeviceListsSince = async function(
-    oldSyncToken, lastKnownSyncToken,
-) {
-    const r = await this._baseApis.getKeyChanges(
-        oldSyncToken, lastKnownSyncToken,
-    );
+Crypto.prototype._evalDeviceListChanges = async function(deviceLists) {
+    if (deviceLists.changed && Array.isArray(deviceLists.changed)) {
+        deviceLists.changed.forEach((u) => {
+            this._deviceList.invalidateUserDeviceList(u);
+        });
+    }

-    console.log("got key changes since", oldSyncToken, ":", r);
+    if (deviceLists.left && Array.isArray(deviceLists.left) &&
+        deviceLists.left.length) {
+        // Check we really don't share any rooms with these users
+        // any more: the server isn't required to give us the
+        // exact correct set.
+        const e2eUserIds = new Set(await this._getTrackedE2eUsers());

-    await this.handleDeviceListChanges(r);
+        deviceLists.left.forEach((u) => {
+            if (!e2eUserIds.has(u)) {
+                this._deviceList.stopTrackingDeviceList(u);
+            }
+        });
+    }
 };

 /**
- * Get a list of the e2e-enabled rooms we are members of
+ * Get a list of all the IDs of users we share an e2e room with
+ * for which we are tracking devices already
+ *
+ * @returns {string[]} List of user IDs
+ */
+Crypto.prototype._getTrackedE2eUsers = async function() {
+    const e2eUserIds = [];
+    for (const room of this._getTrackedE2eRooms()) {
+        const members = await room.getEncryptionTargetMembers();
+        for (const member of members) {
+            e2eUserIds.push(member.userId);
+        }
+    }
+    return e2eUserIds;
+};
+
+/**
+ * Get a list of the e2e-enabled rooms we are members of,
+ * and for which we are already tracking the devices
 *
 * @returns {module:models.Room[]}
 */
-Crypto.prototype._getE2eRooms = function() {
+Crypto.prototype._getTrackedE2eRooms = function() {
    return this._clientStore.getRooms().filter((room) => {
        // check for rooms with encryption enabled
        const alg = this._roomEncryptors[room.roomId];
        if (!alg) {
            return false;
        }
-
-        // ignore any rooms which we have left
-        const me = room.getMember(this._userId);
-        if (!me || (
-            me.membership !== "join" && me.membership !== "invite"
-        )) {
+        if (!this._roomDeviceTrackingState[room.roomId]) {
            return false;
        }

-        return true;
+        // ignore any rooms which we have left
+        const myMembership = room.getMyMembership();
+        return myMembership === "join" || myMembership === "invite";
    });
 };

@@ -966,7 +1140,7 @@ Crypto.prototype._onToDeviceEvent = function(event) {
            });
        }
    } catch (e) {
-        console.error("Error handling toDeviceEvent:", e);
+        logger.error("Error handling toDeviceEvent:", e);
    }
 };

@@ -980,7 +1154,7 @@ Crypto.prototype._onRoomKeyEvent = function(event) {
    const content = event.getContent();

    if (!content.room_id || !content.algorithm) {
-        console.error("key event is missing fields");
+        logger.error("key event is missing fields");
        return;
    }

@@ -1012,11 +1186,20 @@ Crypto.prototype._onRoomMembership = function(event, member, oldMembership) {
        // not encrypting in this room
        return;
    }
-
-    if (member.membership == 'join') {
-        console.log('Join event for ' + member.userId + ' in ' + roomId);
-        // make sure we are tracking the deviceList for this user
-        this._deviceList.startTrackingDeviceList(member.userId);
+    // only mark users in this room as tracked if we already started tracking in this room
+    // this way we don't start device queries after sync on behalf of this room which we won't use
+    // the result of anyway, as we'll need to do a query again once all the members are fetched
+    // by calling _trackRoomDevices
+    if (this._roomDeviceTrackingState[roomId]) {
+        if (member.membership == 'join') {
+            logger.log('Join event for ' + member.userId + ' in ' + roomId);
+            // make sure we are tracking the deviceList for this user
+            this._deviceList.startTrackingDeviceList(member.userId);
+        } else if (member.membership == 'invite' &&
+                 this._clientStore.getRoom(roomId).shouldEncryptForInvitedMembers()) {
+            logger.log('Invite event for ' + member.userId + ' in ' + roomId);
+            this._deviceList.startTrackingDeviceList(member.userId);
+        }
    }

    alg.onRoomMembership(event, member, oldMembership);
@@ -1081,7 +1264,7 @@ Crypto.prototype._processReceivedRoomKeyRequests = async function() {
                this._processReceivedRoomKeyRequestCancellation(cancellation),
        );
    } catch (e) {
-        console.error(`Error processing room key requsts: ${e}`);
+        logger.error(`Error processing room key requsts: ${e}`);
    } finally {
        this._processingRoomKeyRequests = false;
    }
@@ -1100,13 +1283,13 @@ Crypto.prototype._processReceivedRoomKeyRequest = async function(req) {
    const roomId = body.room_id;
    const alg = body.algorithm;

-    console.log(`m.room_key_request from ${userId}:${deviceId}` +
+    logger.log(`m.room_key_request from ${userId}:${deviceId}` +
                ` for ${roomId} / ${body.session_id} (id ${req.requestId})`);

    if (userId !== this._userId) {
        // TODO: determine if we sent this device the keys already: in
        // which case we can do so again.
-        console.log("Ignoring room key request from other user for now");
+        logger.log("Ignoring room key request from other user for now");
        return;
    }

@@ -1116,18 +1299,18 @@ Crypto.prototype._processReceivedRoomKeyRequest = async function(req) {
    // if we don't have a decryptor for this room/alg, we don't have
    // the keys for the requested events, and can drop the requests.
    if (!this._roomDecryptors[roomId]) {
-        console.log(`room key request for unencrypted room ${roomId}`);
+        logger.log(`room key request for unencrypted room ${roomId}`);
        return;
    }

    const decryptor = this._roomDecryptors[roomId][alg];
    if (!decryptor) {
-        console.log(`room key request for unknown alg ${alg} in room ${roomId}`);
+        logger.log(`room key request for unknown alg ${alg} in room ${roomId}`);
        return;
    }

    if (!await decryptor.hasKeysForKeyRequest(req)) {
-        console.log(
+        logger.log(
            `room key request for unknown session ${roomId} / ` +
                body.session_id,
        );
@@ -1141,7 +1324,7 @@ Crypto.prototype._processReceivedRoomKeyRequest = async function(req) {
    // if the device is is verified already, share the keys
    const device = this._deviceList.getStoredDevice(userId, deviceId);
    if (device && device.isVerified()) {
-        console.log('device is already verified: sharing keys');
+        logger.log('device is already verified: sharing keys');
        req.share();
        return;
    }
@@ -1158,7 +1341,7 @@ Crypto.prototype._processReceivedRoomKeyRequest = async function(req) {
 Crypto.prototype._processReceivedRoomKeyRequestCancellation = async function(
    cancellation,
 ) {
-    console.log(
+    logger.log(
        `m.room_key_request cancellation for ${cancellation.userId}:` +
            `${cancellation.deviceId} (id ${cancellation.requestId})`,
    );
@@ -1207,6 +1390,7 @@ Crypto.prototype._getRoomDecryptor = function(roomId, algorithm) {
    const AlgClass = algorithms.DECRYPTION_CLASSES[algorithm];
    if (!AlgClass) {
        throw new algorithms.DecryptionError(
+            'UNKNOWN_ENCRYPTION_ALGORITHM',
            'Unknown encryption algorithm "' + algorithm + '".',
        );
    }
@@ -1324,5 +1508,21 @@ class IncomingRoomKeyRequestCancellation {
 * @param {module:crypto~IncomingRoomKeyRequestCancellation} req
 */

-/** */
-module.exports = Crypto;
+/**
+ * Fires when the app may wish to warn the user about something related
+ * the end-to-end crypto.
+ *
+ * Comes with a type which is one of:
+ * * CRYPTO_WARNING_ACCOUNT_MIGRATED: Account data has been migrated from an older
+ *       version of the store in such a way that older clients will no longer be
+ *       able to read it. The app may wish to warn the user against going back to
+ *       an older version of the app.
+ * * CRYPTO_WARNING_OLD_VERSION_DETECTED: js-sdk has detected that an older version
+ *       of js-sdk has been run against the same store after a migration has been
+ *       performed. This is likely have caused unexpected behaviour in the old
+ *       version. For example, the old version and the new version may have two
+ *       different identity keys.
+ *
+ * @event module:client~MatrixClient#"crypto.warning"
+ * @param {string} type One of the strings listed above
+ */
@@ -23,6 +23,7 @@ limitations under the License.
 import Promise from 'bluebird';
 const anotherjson = require('another-json');

+const logger = require("../logger");
 const utils = require("../utils");

 /**
@@ -65,7 +66,7 @@ module.exports.encryptMessageForDevice = async function(
        return;
    }

-    console.log(
+    logger.log(
        "Using sessionid " + sessionId + " for device " +
            recipientUserId + ":" + recipientDevice.deviceId,
    );
@@ -190,7 +191,7 @@ module.exports.ensureOlmSessionsForDevices = async function(
            }

            if (!oneTimeKey) {
-                console.warn(
+                logger.warn(
                    "No one-time keys (alg=" + oneTimeKeyAlgorithm +
                        ") for device " + userId + ":" + deviceId,
                );
@@ -219,7 +220,7 @@ async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceIn
            deviceInfo.getFingerprint(),
        );
    } catch (e) {
-        console.error(
+        logger.error(
            "Unable to verify signature on one-time key for device " +
                userId + ":" + deviceId + ":", e,
        );
@@ -233,12 +234,12 @@ async function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceIn
        );
    } catch (e) {
        // possibly a bad key
-        console.error("Error starting session with device " +
+        logger.error("Error starting session with device " +
                      userId + ":" + deviceId + ": " + e);
        return null;
    }

-    console.log("Started new sessionid " + sid +
+    logger.log("Started new sessionid " + sid +
                " for device " + userId + ":" + deviceId);
    return sid;
 }
@@ -1,7 +1,26 @@
+/*
+Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import Promise from 'bluebird';
+
+import logger from '../../logger';
 import utils from '../../utils';

-export const VERSION = 1;
+export const VERSION = 6;

 /**
 * Implementation of a CryptoStore which is backed by an existing
@@ -21,7 +40,7 @@ export class Backend {
        // attempts to delete the database will block (and subsequent
        // attempts to re-create it will also block).
        db.onversionchange = (ev) => {
-            console.log(`versionchange for indexeddb ${this._dbName}: closing`);
+            logger.log(`versionchange for indexeddb ${this._dbName}: closing`);
            db.close();
        };
    }
@@ -47,7 +66,7 @@ export class Backend {
        this._getOutgoingRoomKeyRequest(txn, requestBody, (existing) => {
            if (existing) {
                // this entry matches the request - return it.
-                console.log(
+                logger.log(
                    `already have key request outstanding for ` +
                        `${requestBody.room_id} / ${requestBody.session_id}: ` +
                        `not sending another`,
@@ -58,13 +77,13 @@ export class Backend {

            // we got to the end of the list without finding a match
            // - add the new request.
-            console.log(
+            logger.log(
                `enqueueing key request for ${requestBody.room_id} / ` +
                    requestBody.session_id,
            );
+            txn.oncomplete = () => { deferred.resolve(request); };
            const store = txn.objectStore("outgoingRoomKeyRequests");
            store.add(request);
-            txn.onsuccess = () => { deferred.resolve(request); };
        });

        return deferred.promise;
@@ -209,7 +228,7 @@ export class Backend {
            }
            const data = cursor.value;
            if (data.state != expectedState) {
-                console.warn(
+                logger.warn(
                    `Cannot update room key request from ${expectedState} ` +
                    `as it was already updated to ${data.state}`,
                );
@@ -247,7 +266,7 @@ export class Backend {
            }
            const data = cursor.value;
            if (data.state != expectedState) {
-                console.warn(
+                logger.warn(
                    `Cannot delete room key request in state ${data.state} `
                        + `(expected ${expectedState})`,
                );
@@ -257,16 +276,230 @@ export class Backend {
        };
        return promiseifyTxn(txn);
    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeAccount(txn, newData) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(newData, "-");
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const countReq = objectStore.count();
+        countReq.onsuccess = function() {
+            func(countReq.result);
+        };
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const idx = objectStore.index("deviceKey");
+        const getReq = idx.openCursor(deviceKey);
+        const results = {};
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                results[cursor.value.sessionId] = cursor.value.session;
+                cursor.continue();
+            } else {
+                try {
+                    func(results);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const objectStore = txn.objectStore("sessions");
+        const getReq = objectStore.get([deviceKey, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const objectStore = txn.objectStore("sessions");
+        objectStore.put({deviceKey, sessionId, session});
+    }
+
+    // Inbound group sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.get([senderCurve25519Key, sessionId]);
+        getReq.onsuccess = function() {
+            try {
+                if (getReq.result) {
+                    func(getReq.result.session);
+                } else {
+                    func(null);
+                }
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                try {
+                    func({
+                        senderKey: cursor.value.senderCurve25519Key,
+                        sessionId: cursor.value.sessionId,
+                        sessionData: cursor.value.session,
+                    });
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+                cursor.continue();
+            } else {
+                try {
+                    func(null);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        const addReq = objectStore.add({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+        addReq.onerror = (ev) => {
+            if (addReq.error.name === 'ConstraintError') {
+                // This stops the error from triggering the txn's onerror
+                ev.stopPropagation();
+                // ...and this stops it from aborting the transaction
+                ev.preventDefault();
+                logger.log(
+                    "Ignoring duplicate inbound group session: " +
+                    senderCurve25519Key + " / " + sessionId,
+                );
+            } else {
+                abortWithException(txn, new Error(
+                    "Failed to add inbound group session: " + addReq.error,
+                ));
+            }
+        };
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const objectStore = txn.objectStore("inbound_group_sessions");
+        objectStore.put({
+            senderCurve25519Key, sessionId, session: sessionData,
+        });
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        const objectStore = txn.objectStore("device_data");
+        const getReq = objectStore.get("-");
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        const objectStore = txn.objectStore("device_data");
+        objectStore.put(deviceData, "-");
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        const objectStore = txn.objectStore("rooms");
+        objectStore.put(roomInfo, roomId);
+    }
+
+    getEndToEndRooms(txn, func) {
+        const rooms = {};
+        const objectStore = txn.objectStore("rooms");
+        const getReq = objectStore.openCursor();
+        getReq.onsuccess = function() {
+            const cursor = getReq.result;
+            if (cursor) {
+                rooms[cursor.key] = cursor.value;
+                cursor.continue();
+            } else {
+                try {
+                    func(rooms);
+                } catch (e) {
+                    abortWithException(txn, e);
+                }
+            }
+        };
+    }
+
+    doTxn(mode, stores, func) {
+        const txn = this._db.transaction(stores, mode);
+        const promise = promiseifyTxn(txn);
+        const result = func(txn);
+        return promise.then(() => {
+            return result;
+        });
+    }
 }

 export function upgradeDatabase(db, oldVersion) {
-    console.log(
+    logger.log(
        `Upgrading IndexedDBCryptoStore from version ${oldVersion}`
            + ` to ${VERSION}`,
    );
    if (oldVersion < 1) { // The database did not previously exist.
        createDatabase(db);
    }
+    if (oldVersion < 2) {
+        db.createObjectStore("account");
+    }
+    if (oldVersion < 3) {
+        const sessionsStore = db.createObjectStore("sessions", {
+            keyPath: ["deviceKey", "sessionId"],
+        });
+        sessionsStore.createIndex("deviceKey", "deviceKey");
+    }
+    if (oldVersion < 4) {
+        db.createObjectStore("inbound_group_sessions", {
+            keyPath: ["senderCurve25519Key", "sessionId"],
+        });
+    }
+    if (oldVersion < 5) {
+        db.createObjectStore("device_data");
+    }
+    if (oldVersion < 6) {
+        db.createObjectStore("rooms");
+    }
    // Expand as needed.
 }

@@ -283,9 +516,37 @@ function createDatabase(db) {
    outgoingRoomKeyRequestsStore.createIndex("state", "state");
 }

+/*
+ * Aborts a transaction with a given exception
+ * The transaction promise will be rejected with this exception.
+ */
+function abortWithException(txn, e) {
+    // We cheekily stick our exception onto the transaction object here
+    // We could alternatively make the thing we pass back to the app
+    // an object containing the transaction and exception.
+    txn._mx_abortexception = e;
+    try {
+        txn.abort();
+    } catch (e) {
+        // sometimes we won't be able to abort the transaction
+        // (ie. if it's aborted or completed)
+    }
+}
+
 function promiseifyTxn(txn) {
    return new Promise((resolve, reject) => {
-        txn.oncomplete = resolve;
-        txn.onerror = reject;
+        txn.oncomplete = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            resolve();
+        };
+        txn.onerror = () => {
+            if (txn._mx_abortexception !== undefined) {
+                reject(txn._mx_abortexception);
+            }
+            reject();
+        };
+        txn.onabort = () => reject(txn._mx_abortexception);
    });
 }
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +17,8 @@ limitations under the License.

 import Promise from 'bluebird';

+import logger from '../../logger';
+import LocalStorageCryptoStore from './localStorage-crypto-store';
 import MemoryCryptoStore from './memory-crypto-store';
 import * as IndexedDBCryptoStoreBackend from './indexeddb-crypto-store-backend';

@@ -46,7 +49,7 @@ export default class IndexedDBCryptoStore {

    /**
     * Ensure the database exists and is up-to-date, or fall back to
-     * an in-memory store.
+     * a local storage or in-memory store.
     *
     * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
     * or a MemoryCryptoStore
@@ -62,7 +65,7 @@ export default class IndexedDBCryptoStore {
                return;
            }

-            console.log(`connecting to indexeddb ${this._dbName}`);
+            logger.log(`connecting to indexeddb ${this._dbName}`);

            const req = this._indexedDB.open(
                this._dbName, IndexedDBCryptoStoreBackend.VERSION,
@@ -75,7 +78,7 @@ export default class IndexedDBCryptoStore {
            };

            req.onblocked = () => {
-                console.log(
+                logger.log(
                    `can't yet open IndexedDBCryptoStore because it is open elsewhere`,
                );
            };
@@ -87,13 +90,31 @@ export default class IndexedDBCryptoStore {
            req.onsuccess = (r) => {
                const db = r.target.result;

-                console.log(`connected to indexeddb ${this._dbName}`);
+                logger.log(`connected to indexeddb ${this._dbName}`);
                resolve(new IndexedDBCryptoStoreBackend.Backend(db));
            };
+        }).then((backend) => {
+            // Edge has IndexedDB but doesn't support compund keys which we use fairly extensively.
+            // Try a dummy query which will fail if the browser doesn't support compund keys, so
+            // we can fall back to a different backend.
+            return backend.doTxn(
+                'readonly',
+                [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS],
+                (txn) => {
+                    backend.getEndToEndInboundGroupSession('', '', txn, () => {});
+                }).then(() => {
+                    return backend;
+                },
+            );
        }).catch((e) => {
-            console.warn(
+            logger.warn(
                `unable to connect to indexeddb ${this._dbName}` +
-                    `: falling back to in-memory store: ${e}`,
+                    `: falling back to localStorage store: ${e}`,
+            );
+            return new LocalStorageCryptoStore(global.localStorage);
+        }).catch((e) => {
+            logger.warn(
+                `unable to open localStorage: falling back to in-memory store: ${e}`,
            );
            return new MemoryCryptoStore();
        });
@@ -113,11 +134,11 @@ export default class IndexedDBCryptoStore {
                return;
            }

-            console.log(`Removing indexeddb instance: ${this._dbName}`);
+            logger.log(`Removing indexeddb instance: ${this._dbName}`);
            const req = this._indexedDB.deleteDatabase(this._dbName);

            req.onblocked = () => {
-                console.log(
+                logger.log(
                    `can't yet delete IndexedDBCryptoStore because it is open elsewhere`,
                );
            };
@@ -127,14 +148,14 @@ export default class IndexedDBCryptoStore {
            };

            req.onsuccess = () => {
-                console.log(`Removed indexeddb instance: ${this._dbName}`);
+                logger.log(`Removed indexeddb instance: ${this._dbName}`);
                resolve();
            };
        }).catch((e) => {
            // in firefox, with indexedDB disabled, this fails with a
            // DOMError. We treat this as non-fatal, so that people can
            // still use the app.
-            console.warn(`unable to delete IndexedDBCryptoStore: ${e}`);
+            logger.warn(`unable to delete IndexedDBCryptoStore: ${e}`);
        });
    }

@@ -220,4 +241,216 @@ export default class IndexedDBCryptoStore {
            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
        });
    }
+
+    // Olm Account
+
+    /*
+     * Get the account pickle from the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the account pickle
+     */
+    getAccount(txn, func) {
+        this._backendPromise.value().getAccount(txn, func);
+    }
+
+    /*
+     * Write the account pickle to the store.
+     * This requires an active transaction. See doTxn().
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} newData The new account pickle to store.
+     */
+    storeAccount(txn, newData) {
+        this._backendPromise.value().storeAccount(txn, newData);
+    }
+
+    // Olm sessions
+
+    /**
+     * Returns the number of end-to-end sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(int)} func Called with the count of sessions
+     */
+    countEndToEndSessions(txn, func) {
+        this._backendPromise.value().countEndToEndSessions(txn, func);
+    }
+
+    /**
+     * Retrieve a specific end-to-end session between the logged-in user
+     * and another device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID of the session to retrieve
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndSession(deviceKey, sessionId, txn, func);
+    }
+
+    /**
+     * Retrieve the end-to-end sessions between the logged-in user and another
+     * device.
+     * @param {string} deviceKey The public key of the other device.
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndSessions(deviceKey, txn, func) {
+        this._backendPromise.value().getEndToEndSessions(deviceKey, txn, func);
+    }
+
+    /**
+     * Store a session between the logged-in user and another device
+     * @param {string} deviceKey The public key of the other device.
+     * @param {string} sessionId The ID for this end-to-end session.
+     * @param {string} session Base64 encoded end-to-end session.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        this._backendPromise.value().storeEndToEndSession(
+            deviceKey, sessionId, session, txn,
+        );
+    }
+
+    // Inbound group saessions
+
+    /**
+     * Retrieve the end-to-end inbound group session for a given
+     * server key and session ID
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called with A map from sessionId
+     *     to Base64 end-to-end session.
+     */
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        this._backendPromise.value().getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
+    }
+
+    /**
+     * Fetches all inbound group sessions in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(object)} func Called once for each group session
+     *     in the store with an object having keys {senderKey, sessionId,
+     *     sessionData}, then once with null to indicate the end of the list.
+     */
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        this._backendPromise.value().getAllEndToEndInboundGroupSessions(txn, func);
+    }
+
+    /**
+     * Adds an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, the session will not be added.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    /**
+     * Writes an end-to-end inbound group session to the store.
+     * If there already exists an inbound group session with the same
+     * senderCurve25519Key and sessionID, it will be overwritten.
+     * @param {string} senderCurve25519Key The sender's curve 25519 key
+     * @param {string} sessionId The ID of the session
+     * @param {object} sessionData The session data structure
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._backendPromise.value().storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
+    }
+
+    // End-to-end device tracking
+
+    /**
+     * Store the state of all tracked devices
+     * This contains devices for each user, a tracking state for each user
+     * and a sync token matching the point in time the snapshot represents.
+     * These all need to be written out in full each time such that the snapshot
+     * is always consistent, so they are stored in one object.
+     *
+     * @param {Object} deviceData
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._backendPromise.value().storeEndToEndDeviceData(deviceData, txn);
+    }
+
+    /**
+     * Get the state of all tracked devices
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the
+     *     device data
+     */
+    getEndToEndDeviceData(txn, func) {
+        this._backendPromise.value().getEndToEndDeviceData(txn, func);
+    }
+
+    // End to End Rooms
+
+    /**
+     * Store the end-to-end state for a room.
+     * @param {string} roomId The room's ID.
+     * @param {object} roomInfo The end-to-end info for the room.
+     * @param {*} txn An active transaction. See doTxn().
+     */
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._backendPromise.value().storeEndToEndRoom(roomId, roomInfo, txn);
+    }
+
+    /**
+     * Get an object of roomId->roomInfo for all e2e rooms in the store
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(Object)} func Function called with the end to end encrypted rooms
+     */
+    getEndToEndRooms(txn, func) {
+        this._backendPromise.value().getEndToEndRooms(txn, func);
+    }
+
+    /**
+     * Perform a transaction on the crypto store. Any store methods
+     * that require a transaction (txn) object to be passed in may
+     * only be called within a callback of either this function or
+     * one of the store functions operating on the same transaction.
+     *
+     * @param {string} mode 'readwrite' if you need to call setter
+     *     functions with this transaction. Otherwise, 'readonly'.
+     * @param {string[]} stores List IndexedDBCryptoStore.STORE_*
+     *     options representing all types of object that will be
+     *     accessed or written to with this transaction.
+     * @param {function(*)} func Function called with the
+     *     transaction object: an opaque object that should be passed
+     *     to store functions.
+     * @return {Promise} Promise that resolves with the result of the `func`
+     *     when the transaction is complete. If the backend is
+     *     async (ie. the indexeddb backend) any of the callback
+     *     functions throwing an exception will cause this promise to
+     *     reject with that exception. On synchronous backends, the
+     *     exception will propagate to the caller of the getFoo method.
+     */
+    doTxn(mode, stores, func) {
+        return this._connect().then((backend) => {
+            return backend.doTxn(mode, stores, func);
+        });
+    }
 }
+
+IndexedDBCryptoStore.STORE_ACCOUNT = 'account';
+IndexedDBCryptoStore.STORE_SESSIONS = 'sessions';
+IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS = 'inbound_group_sessions';
+IndexedDBCryptoStore.STORE_DEVICE_DATA = 'device_data';
+IndexedDBCryptoStore.STORE_ROOMS = 'rooms';
@@ -0,0 +1,212 @@
+/*
+Copyright 2017, 2018 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Promise from 'bluebird';
+
+import logger from '../../logger';
+import MemoryCryptoStore from './memory-crypto-store.js';
+
+/**
+ * Internal module. Partial localStorage backed storage for e2e.
+ * This is not a full crypto store, just the in-memory store with
+ * some things backed by localStorage. It exists because indexedDB
+ * is broken in Firefox private mode or set to, "will not remember
+ * history".
+ *
+ * @module
+ */
+
+const E2E_PREFIX = "crypto.";
+const KEY_END_TO_END_ACCOUNT = E2E_PREFIX + "account";
+const KEY_DEVICE_DATA = E2E_PREFIX + "device_data";
+const KEY_INBOUND_SESSION_PREFIX = E2E_PREFIX + "inboundgroupsessions/";
+const KEY_ROOMS_PREFIX = E2E_PREFIX + "rooms/";
+
+function keyEndToEndSessions(deviceKey) {
+    return E2E_PREFIX + "sessions/" + deviceKey;
+}
+
+function keyEndToEndInboundGroupSession(senderKey, sessionId) {
+    return KEY_INBOUND_SESSION_PREFIX + senderKey + "/" + sessionId;
+}
+
+function keyEndToEndRoomsPrefix(roomId) {
+    return KEY_ROOMS_PREFIX + roomId;
+}
+
+/**
+ * @implements {module:crypto/store/base~CryptoStore}
+ */
+export default class LocalStorageCryptoStore extends MemoryCryptoStore {
+    constructor(webStore) {
+        super();
+        this.store = webStore;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        let count = 0;
+        for (let i = 0; i < this.store.length; ++i) {
+            if (this.store.key(i).startsWith(keyEndToEndSessions(''))) ++count;
+        }
+        func(count);
+    }
+
+    _getEndToEndSessions(deviceKey, txn, func) {
+        return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const sessions = this._getEndToEndSessions(deviceKey);
+        func(sessions[sessionId] || {});
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._getEndToEndSessions(deviceKey) || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        const sessions = this._getEndToEndSessions(deviceKey) || {};
+        sessions[sessionId] = session;
+        setJsonItem(
+            this.store, keyEndToEndSessions(deviceKey), sessions,
+        );
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        ));
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(KEY_INBOUND_SESSION_PREFIX)) {
+                // we can't use split, as the components we are trying to split out
+                // might themselves contain '/' characters. We rely on the
+                // senderKey being a (32-byte) curve25519 key, base64-encoded
+                // (hence 43 characters long).
+
+                func({
+                    senderKey: key.substr(KEY_INBOUND_SESSION_PREFIX.length, 43),
+                    sessionId: key.substr(KEY_INBOUND_SESSION_PREFIX.length + 44),
+                    sessionData: getJsonItem(this.store, key),
+                });
+            }
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const existing = getJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+        );
+        if (!existing) {
+            this.storeEndToEndInboundGroupSession(
+                senderCurve25519Key, sessionId, sessionData, txn,
+            );
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        setJsonItem(
+            this.store,
+            keyEndToEndInboundGroupSession(senderCurve25519Key, sessionId),
+            sessionData,
+        );
+    }
+
+    getEndToEndDeviceData(txn, func) {
+        func(getJsonItem(
+            this.store, KEY_DEVICE_DATA,
+        ));
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        setJsonItem(
+            this.store, KEY_DEVICE_DATA, deviceData,
+        );
+    }
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        setJsonItem(
+            this.store, keyEndToEndRoomsPrefix(roomId), roomInfo,
+        );
+    }
+
+    getEndToEndRooms(txn, func) {
+        const result = {};
+        const prefix = keyEndToEndRoomsPrefix('');
+
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            if (key.startsWith(prefix)) {
+                const roomId = key.substr(prefix.length);
+                result[roomId] = getJsonItem(this.store, key);
+            }
+        }
+        func(result);
+    }
+
+    /**
+     * Delete all data from this store.
+     *
+     * @returns {Promise} Promise which resolves when the store has been cleared.
+     */
+    deleteAllData() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
+        return Promise.resolve();
+    }
+
+    // Olm account
+
+    getAccount(txn, func) {
+        const account = getJsonItem(this.store, KEY_END_TO_END_ACCOUNT);
+        func(account);
+    }
+
+    storeAccount(txn, newData) {
+        setJsonItem(
+            this.store, KEY_END_TO_END_ACCOUNT, newData,
+        );
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
+}
+
+function getJsonItem(store, key) {
+    try {
+        // if the key is absent, store.getItem() returns null, and
+        // JSON.parse(null) === null, so this returns null.
+        return JSON.parse(store.getItem(key));
+    } catch (e) {
+        logger.log("Error: Failed to get key %s: %s", key, e.stack || e);
+        logger.log(e.stack);
+    }
+    return null;
+}
+
+function setJsonItem(store, key, val) {
+    store.setItem(key, JSON.stringify(val));
+}
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +17,7 @@ limitations under the License.

 import Promise from 'bluebird';

+import logger from '../../logger';
 import utils from '../../utils';

 /**
@@ -30,6 +32,16 @@ import utils from '../../utils';
 export default class MemoryCryptoStore {
    constructor() {
        this._outgoingRoomKeyRequests = [];
+        this._account = null;
+
+        // Map of {devicekey -> {sessionId -> session pickle}}
+        this._sessions = {};
+        // Map of {senderCurve25519Key+'/'+sessionId -> session data object}
+        this._inboundGroupSessions = {};
+        // Opaque device data object
+        this._deviceData = null;
+        // roomId -> Opaque roomInfo object
+        this._rooms = {};
    }

    /**
@@ -60,7 +72,7 @@ export default class MemoryCryptoStore {

            if (existing) {
                // this entry matches the request - return it.
-                console.log(
+                logger.log(
                    `already have key request outstanding for ` +
                    `${requestBody.room_id} / ${requestBody.session_id}: ` +
                    `not sending another`,
@@ -70,7 +82,7 @@ export default class MemoryCryptoStore {

            // we got to the end of the list without finding a match
            // - add the new request.
-            console.log(
+            logger.log(
                `enqueueing key request for ${requestBody.room_id} / ` +
                requestBody.session_id,
            );
@@ -152,7 +164,7 @@ export default class MemoryCryptoStore {
            }

            if (req.state != expectedState) {
-                console.warn(
+                logger.warn(
                    `Cannot update room key request from ${expectedState} ` +
                    `as it was already updated to ${req.state}`,
                );
@@ -183,7 +195,7 @@ export default class MemoryCryptoStore {
            }

            if (req.state != expectedState) {
-                console.warn(
+                logger.warn(
                    `Cannot delete room key request in state ${req.state} `
                    + `(expected ${expectedState})`,
                );
@@ -196,4 +208,95 @@ export default class MemoryCryptoStore {

        return Promise.resolve(null);
    }
+
+    // Olm Account
+
+    getAccount(txn, func) {
+        func(this._account);
+    }
+
+    storeAccount(txn, newData) {
+        this._account = newData;
+    }
+
+    // Olm Sessions
+
+    countEndToEndSessions(txn, func) {
+        return Object.keys(this._sessions).length;
+    }
+
+    getEndToEndSession(deviceKey, sessionId, txn, func) {
+        const deviceSessions = this._sessions[deviceKey] || {};
+        func(deviceSessions[sessionId] || null);
+    }
+
+    getEndToEndSessions(deviceKey, txn, func) {
+        func(this._sessions[deviceKey] || {});
+    }
+
+    storeEndToEndSession(deviceKey, sessionId, session, txn) {
+        let deviceSessions = this._sessions[deviceKey];
+        if (deviceSessions === undefined) {
+            deviceSessions = {};
+            this._sessions[deviceKey] = deviceSessions;
+        }
+        deviceSessions[sessionId] = session;
+    }
+
+    // Inbound Group Sessions
+
+    getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
+        func(this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] || null);
+    }
+
+    getAllEndToEndInboundGroupSessions(txn, func) {
+        for (const key of Object.keys(this._inboundGroupSessions)) {
+            // we can't use split, as the components we are trying to split out
+            // might themselves contain '/' characters. We rely on the
+            // senderKey being a (32-byte) curve25519 key, base64-encoded
+            // (hence 43 characters long).
+
+            func({
+                senderKey: key.substr(0, 43),
+                sessionId: key.substr(44),
+                sessionData: this._inboundGroupSessions[key],
+            });
+        }
+        func(null);
+    }
+
+    addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        const k = senderCurve25519Key+'/'+sessionId;
+        if (this._inboundGroupSessions[k] === undefined) {
+            this._inboundGroupSessions[k] = sessionData;
+        }
+    }
+
+    storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
+        this._inboundGroupSessions[senderCurve25519Key+'/'+sessionId] = sessionData;
+    }
+
+    // Device Data
+
+    getEndToEndDeviceData(txn, func) {
+        func(this._deviceData);
+    }
+
+    storeEndToEndDeviceData(deviceData, txn) {
+        this._deviceData = deviceData;
+    }
+
+    // E2E rooms
+
+    storeEndToEndRoom(roomId, roomInfo, txn) {
+        this._rooms[roomId] = roomInfo;
+    }
+
+    getEndToEndRooms(txn, func) {
+        func(this._rooms);
+    }
+
+    doTxn(mode, stores, func) {
+        return Promise.resolve(func(null));
+    }
 }
@@ -0,0 +1,25 @@
+// can't just do InvalidStoreError extends Error
+// because of http://babeljs.io/docs/usage/caveats/#classes
+function InvalidStoreError(reason, value) {
+    const message = `Store is invalid because ${reason}, ` +
+        `please stop the client, delete all data and start the client again`;
+    const instance = Reflect.construct(Error, [message]);
+    Reflect.setPrototypeOf(instance, Reflect.getPrototypeOf(this));
+    instance.reason = reason;
+    instance.value = value;
+    return instance;
+}
+
+InvalidStoreError.TOGGLED_LAZY_LOADING = "TOGGLED_LAZY_LOADING";
+
+InvalidStoreError.prototype = Object.create(Error.prototype, {
+  constructor: {
+    value: Error,
+    enumerable: false,
+    writable: true,
+    configurable: true,
+  },
+});
+Reflect.setPrototypeOf(InvalidStoreError, Error);
+
+module.exports.InvalidStoreError = InvalidStoreError;
@@ -97,11 +97,12 @@ FilterComponent.prototype._checkFields =
    };

    const self = this;
-    Object.keys(literal_keys).forEach(function(name) {
+    for (let n=0; n < Object.keys(literal_keys).length; n++) {
+        const name = Object.keys(literal_keys)[n];
        const match_func = literal_keys[name];
        const not_name = "not_" + name;
        const disallowed_values = self[not_name];
-        if (disallowed_values.map(match_func)) {
+        if (disallowed_values.filter(match_func).length > 0) {
            return false;
        }

@@ -111,7 +112,7 @@ FilterComponent.prototype._checkFields =
                return false;
            }
        }
-    });
+    }

    const contains_url_filter = this.filter_json.contains_url;
    if (contains_url_filter !== undefined) {
@@ -51,6 +51,17 @@ function Filter(userId, filterId) {
    this.definition = {};
 }

+Filter.LAZY_LOADING_MESSAGES_FILTER = {
+    lazy_load_members: true,
+};
+
+Filter.LAZY_LOADING_SYNC_FILTER = {
+    room: {
+        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
+    },
+};
+
+
 /**
 * Get the ID of this filter on your homeserver (if known)
 * @return {?Number} The filter ID
@@ -118,6 +118,10 @@ module.exports.MatrixHttpApi.prototype = {
     * @param {string=} opts.name   Name to give the file on the server. Defaults
     *   to <tt>file.name</tt>.
     *
+     * @param {boolean=} opts.includeFilename if false will not send the filename,
+     *   e.g for encrypted file uploads where filename leaks are undesirable.
+     *   Defaults to true.
+     *
     * @param {string=} opts.type   Content-type for the upload. Defaults to
     *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
     *
@@ -152,6 +156,9 @@ module.exports.MatrixHttpApi.prototype = {
            opts = {};
        }

+        // default opts.includeFilename to true (ignoring falsey values)
+        const includeFilename = opts.includeFilename !== false;
+
        // if the file doesn't have a mime type, use a default since
        // the HS errors if we don't supply one.
        const contentType = opts.type || file.type || 'application/octet-stream';
@@ -272,10 +279,26 @@ module.exports.MatrixHttpApi.prototype = {
                }
            });
            let url = this.opts.baseUrl + "/_matrix/media/v1/upload";
-            url += "?access_token=" + encodeURIComponent(this.opts.accessToken);
-            url += "&filename=" + encodeURIComponent(fileName);
+
+            const queryArgs = [];
+
+            if (includeFilename && fileName) {
+                queryArgs.push("filename=" + encodeURIComponent(fileName));
+            }
+
+            if (!this.useAuthorizationHeader) {
+                queryArgs.push("access_token="
+                    + encodeURIComponent(this.opts.accessToken));
+            }
+
+            if (queryArgs.length > 0) {
+                url += "?" + queryArgs.join("&");
+            }

            xhr.open("POST", url);
+            if (this.useAuthorizationHeader) {
+                xhr.setRequestHeader("Authorization", "Bearer " + this.opts.accessToken);
+            }
            xhr.setRequestHeader("Content-Type", contentType);
            xhr.send(body);
            promise = defer.promise;
@@ -283,9 +306,11 @@ module.exports.MatrixHttpApi.prototype = {
            // dirty hack (as per _request) to allow the upload to be cancelled.
            promise.abort = xhr.abort.bind(xhr);
        } else {
-            const queryParams = {
-                filename: fileName,
-            };
+            const queryParams = {};
+
+            if (includeFilename && fileName) {
+                queryParams.filename = fileName;
+            }

            promise = this.authedRequest(
                opts.callback, "POST", "/upload", queryParams, body, {
@@ -433,6 +458,12 @@ module.exports.MatrixHttpApi.prototype = {
        requestPromise.catch(function(err) {
            if (err.errcode == 'M_UNKNOWN_TOKEN') {
                self.event_emitter.emit("Session.logged_out");
+            } else if (err.errcode == 'M_CONSENT_NOT_GIVEN') {
+                self.event_emitter.emit(
+                    "no_consent",
+                    err.message,
+                    err.data.consent_uri,
+                );
            }
        });

@@ -721,10 +752,12 @@ module.exports.MatrixHttpApi.prototype = {
                    method: method,
                    withCredentials: false,
                    qs: queryParams,
+                    qsStringifyOptions: opts.qsStringifyOptions,
+                    useQuerystring: true,
                    body: data,
                    json: false,
                    timeout: localTimeoutMs,
-                    headers: opts.headers || {},
+                    headers: headers || {},
                    _matrix_opts: this.opts,
                },
                function(err, response, body) {
@@ -0,0 +1,36 @@
+/*
+Copyright 2018 André Jaenisch
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * @module logger
+ */
+const log = require("loglevel");
+
+// This is to demonstrate, that you can use any namespace you want.
+// Namespaces allow you to turn on/off the logging for specific parts of the
+// application.
+// An idea would be to control this via an environment variable (on Node.js).
+// See https://www.npmjs.com/package/debug to see how this could be implemented
+// Part of #332 is introducing a logging library in the first place.
+const DEFAULT_NAME_SPACE = "matrix";
+const logger = log.getLogger(DEFAULT_NAME_SPACE);
+logger.setLevel(log.levels.DEBUG);
+
+/**
+ * Drop-in replacement for <code>console</code> using {@link https://www.npmjs.com/package/loglevel|loglevel}.
+ * Can be tailored down to specific use cases if needed.
+*/
+module.exports = logger;
@@ -16,6 +16,8 @@ limitations under the License.
 */
 "use strict";

+/** The {@link module:ContentHelpers} object */
+module.exports.ContentHelpers = require("./content-helpers");
 /** The {@link module:models/event.MatrixEvent|MatrixEvent} class. */
 module.exports.MatrixEvent = require("./models/event").MatrixEvent;
 /** The {@link module:models/event.EventStatus|EventStatus} enum. */
@@ -32,10 +34,14 @@ module.exports.SyncAccumulator = require("./sync-accumulator");
 module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi;
 /** The {@link module:http-api.MatrixError|MatrixError} class. */
 module.exports.MatrixError = require("./http-api").MatrixError;
+/** The {@link module:errors.InvalidStoreError|InvalidStoreError} class. */
+module.exports.InvalidStoreError = require("./errors").InvalidStoreError;
 /** The {@link module:client.MatrixClient|MatrixClient} class. */
 module.exports.MatrixClient = require("./client").MatrixClient;
 /** The {@link module:models/room|Room} class. */
 module.exports.Room = require("./models/room");
+/** The {@link module:models/group|Group} class. */
+module.exports.Group = require("./models/group");
 /** The {@link module:models/event-timeline~EventTimeline} class. */
 module.exports.EventTimeline = require("./models/event-timeline");
 /** The {@link module:models/event-timeline-set~EventTimelineSet} class. */
@@ -79,6 +85,13 @@ module.exports.IndexedDBCryptoStore =
 module.exports.createNewMatrixCall = require("./webrtc/call").createNewMatrixCall;


+/**
+ * Set an audio output device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setMatrixCallAudioOutput = require('./webrtc/call').setAudioOutput;
 /**
 * Set an audio input device to use for MatrixCalls
 * @function
@@ -168,49 +168,19 @@ EventTimelineSet.prototype.resetLiveTimeline = function(
    // if timeline support is disabled, forget about the old timelines
    const resetAllTimelines = !this._timelineSupport || !forwardPaginationToken;

-    let newTimeline;
+    const oldTimeline = this._liveTimeline;
+    const newTimeline = resetAllTimelines ?
+        oldTimeline.forkLive(EventTimeline.FORWARDS) :
+        oldTimeline.fork(EventTimeline.FORWARDS);
+
    if (resetAllTimelines) {
-        newTimeline = new EventTimeline(this);
        this._timelines = [newTimeline];
        this._eventIdToTimeline = {};
    } else {
-        newTimeline = this.addTimeline();
+        this._timelines.push(newTimeline);
    }

-    const oldTimeline = this._liveTimeline;
-
-    // Collect the state events from the old timeline
-    const evMap = oldTimeline.getState(EventTimeline.FORWARDS).events;
-    const events = [];
-    for (const evtype in evMap) {
-        if (!evMap.hasOwnProperty(evtype)) {
-            continue;
-        }
-        for (const stateKey in evMap[evtype]) {
-            if (!evMap[evtype].hasOwnProperty(stateKey)) {
-                continue;
-            }
-            events.push(evMap[evtype][stateKey]);
-        }
-    }
-
-    // Use those events to initialise the state of the new live timeline
-    newTimeline.initialiseState(events);
-
-    const freshEndState = newTimeline._endState;
-    // Now clobber the end state of the new live timeline with that from the
-    // previous live timeline. It will be identical except that we'll keep
-    // using the same RoomMember objects for the 'live' set of members with any
-    // listeners still attached
-    newTimeline._endState = oldTimeline._endState;
-
-    // If we're not resetting all timelines, we need to fix up the old live timeline
-    if (!resetAllTimelines) {
-        // Firstly, we just stole the old timeline's end state, so it needs a new one.
-        // Just swap them around and give it the one we just generated for the
-        // new live timeline.
-        oldTimeline._endState = freshEndState;
-
+    if (forwardPaginationToken) {
        // Now set the forward pagination token on the old live timeline
        // so it can be forward-paginated.
        oldTimeline.setPaginationToken(
@@ -20,8 +20,6 @@ limitations under the License.
 */

 const RoomState = require("./room-state");
-const utils = require("../utils");
-const MatrixEvent = require("./event").MatrixEvent;

 /**
 * Construct a new EventTimeline
@@ -88,22 +86,70 @@ EventTimeline.prototype.initialiseState = function(stateEvents) {
        throw new Error("Cannot initialise state after events are added");
    }

-    // we deep-copy the events here, in case they get changed later - we don't
-    // want changes to the start state leaking through to the end state.
-    const oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEvents.map(function(mxEvent) {
-                return mxEvent.event;
-            }),
-        ),
-    function(ev) {
-        return new MatrixEvent(ev);
-    });
+    // We previously deep copied events here and used different copies in
+    // the oldState and state events: this decision seems to date back
+    // quite a way and was apparently made to fix a bug where modifications
+    // made to the start state leaked through to the end state.
+    // This really shouldn't be possible though: the events themselves should
+    // not change. Duplicating the events uses a lot of extra memory,
+    // so we now no longer do it. To assert that they really do never change,
+    // freeze them! Note that we can't do this for events in general:
+    // although it looks like the only things preventing us are the
+    // 'status' flag, forwardLooking (which is only set once when adding to the
+    // timeline) and possibly the sender (which seems like it should never be
+    // reset but in practice causes a lot of the tests to break).
+    for (const e of stateEvents) {
+        Object.freeze(e);
+    }

-    this._startState.setStateEvents(oldStateEvents);
+    this._startState.setStateEvents(stateEvents);
    this._endState.setStateEvents(stateEvents);
 };

+/**
+ * Forks the (live) timeline, taking ownership of the existing directional state of this timeline.
+ * All attached listeners will keep receiving state updates from the new live timeline state.
+ * The end state of this timeline gets replaced with an independent copy of the current RoomState,
+ * and will need a new pagination token if it ever needs to paginate forwards.
+
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.forkLive = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    // Now clobber the end state of the new live timeline with that from the
+    // previous live timeline. It will be identical except that we'll keep
+    // using the same RoomMember objects for the 'live' set of members with any
+    // listeners still attached
+    timeline._endState = forkState;
+    // Firstly, we just stole the current timeline's end state, so it needs a new one.
+    // Make an immutable copy of the state so back pagination will get the correct sentinels.
+    this._endState = forkState.clone();
+    return timeline;
+};
+
+/**
+ * Creates an independent timeline, inheriting the directional state from this timeline.
+ *
+ * @param {string} direction   EventTimeline.BACKWARDS to get the state at the
+ *   start of the timeline; EventTimeline.FORWARDS to get the state at the end
+ *   of the timeline.
+ *
+ * @return {EventTimeline} the new timeline
+ */
+EventTimeline.prototype.fork = function(direction) {
+    const forkState = this.getState(direction);
+    const timeline = new EventTimeline(this._eventTimelineSet);
+    timeline._startState = forkState.clone();
+    timeline._endState = forkState.clone();
+    return timeline;
+};
+
 /**
 * Get the ID of the room for this timeline
 * @return {string} room ID
@@ -80,7 +80,10 @@ module.exports.MatrixEvent = function MatrixEvent(
    // intern the values of matrix events to force share strings and reduce the
    // amount of needless string duplication. This can save moderate amounts of
    // memory (~10% on a 350MB heap).
-    ["state_key", "type", "sender", "room_id"].forEach((prop) => {
+    // 'membership' at the event level (rather than the content level) is a legacy
+    // field that Riot never otherwise looks at, but it will still take up a lot
+    // of space if we don't intern it.
+    ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
        if (!event[prop]) {
            return;
        }
@@ -108,8 +111,6 @@ module.exports.MatrixEvent = function MatrixEvent(
    this.error = null;
    this.forwardLooking = true;
    this._pushActions = null;
-    this._date = this.event.origin_server_ts ?
-        new Date(this.event.origin_server_ts) : null;

    this._clearEvent = {};

@@ -204,7 +205,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     * @return {Date} The event date, e.g. <code>new Date(1433502692297)</code>
     */
    getDate: function() {
-        return this._date;
+        return this.event.origin_server_ts ? new Date(this.event.origin_server_ts) : null;
    },

    /**
@@ -377,6 +378,21 @@ utils.extend(module.exports.MatrixEvent.prototype, {
        return this._decryptionPromise;
    },

+    /**
+     * Cancel any room key request for this event and resend another.
+     *
+     * @param {module:crypto} crypto crypto module
+     */
+    cancelAndResendKeyRequest: function(crypto) {
+        const wireContent = this.getWireContent();
+        crypto.cancelRoomKeyRequest({
+            algorithm: wireContent.algorithm,
+            room_id: this.getRoomId(),
+            session_id: wireContent.session_id,
+            sender_key: wireContent.sender_key,
+        }, true);
+    },
+
    _decryptionLoop: async function(crypto) {
        // make sure that this method never runs completely synchronously.
        // (doing so would mean that we would clear _decryptionPromise *before*
@@ -388,6 +404,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
            this._retryDecryption = false;

            let res;
+            let err;
            try {
                if (!crypto) {
                    res = this._badEncryptedMessage("Encryption not enabled");
@@ -406,6 +423,8 @@ utils.extend(module.exports.MatrixEvent.prototype, {
                    return;
                }

+                err = e;
+
                // see if we have a retry queued.
                //
                // NB: make sure to keep this check in the same tick of the
@@ -424,7 +443,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
                    // decryption error, but we have a retry queued.
                    console.log(
                        `Got error decrypting event (id=${this.getId()}: ` +
-                        `${e.message}), but retrying`,
+                        `${e}), but retrying`,
                    );
                    continue;
                }
@@ -432,7 +451,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
                // decryption error, no retries queued. Warn about the error and
                // set it to m.bad.encrypted.
                console.warn(
-                    `Error decrypting event (id=${this.getId()}): ${e}`,
+                    `Error decrypting event (id=${this.getId()}): ${e.detailedString}`,
                );

                res = this._badEncryptedMessage(e.message);
@@ -451,6 +470,9 @@ utils.extend(module.exports.MatrixEvent.prototype, {
            this._decryptionPromise = null;
            this._retryDecryption = false;
            this._setClearData(res);
+
+            this.emit("Event.decrypted", this, err);
+
            return;
        }
    },
@@ -487,7 +509,6 @@ utils.extend(module.exports.MatrixEvent.prototype, {
            decryptionResult.claimedEd25519Key || null;
        this._forwardingCurve25519KeyChain =
            decryptionResult.forwardingCurve25519KeyChain || [];
-        this.emit("Event.decrypted", this);
    },

    /**
@@ -652,7 +673,6 @@ utils.extend(module.exports.MatrixEvent.prototype, {
        this.event = event;
        // successfully sent.
        this.status = null;
-        this._date = new Date(this.event.origin_server_ts);
     },
 });

@@ -693,4 +713,7 @@ const _REDACT_KEEP_CONTENT_MAP = {
 *
 * @param {module:models/event.MatrixEvent} event
 *    The matrix event which has been decrypted
+ * @param {module:crypto/algorithms/base.DecryptionError?} err
+ *    The error that occured during decryption, or `undefined` if no
+ *    error occured.
 */
@@ -58,10 +58,27 @@ function RoomMember(roomId, userId) {
    this.events = {
        member: null,
    };
+    this._isOutOfBand = false;
    this._updateModifiedTime();
 }
 utils.inherits(RoomMember, EventEmitter);

+/**
+ * Mark the member as coming from a channel that is not sync
+ */
+RoomMember.prototype.markOutOfBand = function() {
+    this._isOutOfBand = true;
+};
+
+/**
+ * @return {bool} does the member come from a channel that is not sync?
+ * This is used to store the member seperately
+ * from the sync state so it available across browser sessions.
+ */
+RoomMember.prototype.isOutOfBand = function() {
+    return this._isOutOfBand;
+};
+
 /**
 * Update this room member's membership event. May fire "RoomMember.name" if
 * this event updates this member's name.
@@ -75,13 +92,20 @@ RoomMember.prototype.setMembershipEvent = function(event, roomState) {
    if (event.getType() !== "m.room.member") {
        return;
    }
+
+    this._isOutOfBand = false;
+
    this.events.member = event;

    const oldMembership = this.membership;
    this.membership = event.getDirectionalContent().membership;

    const oldName = this.name;
-    this.name = calculateDisplayName(this, event, roomState);
+    this.name = calculateDisplayName(
+        this.userId,
+        event.getDirectionalContent().displayname,
+        roomState);
+
    this.rawDisplayName = event.getDirectionalContent().displayname || this.userId;
    if (oldMembership !== this.membership) {
        this._updateModifiedTime();
@@ -104,17 +128,20 @@ RoomMember.prototype.setPowerLevelEvent = function(powerLevelEvent) {
    if (powerLevelEvent.getType() !== "m.room.power_levels") {
        return;
    }
-    let maxLevel = powerLevelEvent.getContent().users_default || 0;
-    utils.forEach(utils.values(powerLevelEvent.getContent().users), function(lvl) {
+
+    const evContent = powerLevelEvent.getDirectionalContent();
+
+    let maxLevel = evContent.users_default || 0;
+    utils.forEach(utils.values(evContent.users), function(lvl) {
        maxLevel = Math.max(maxLevel, lvl);
    });
    const oldPowerLevel = this.powerLevel;
    const oldPowerLevelNorm = this.powerLevelNorm;

-    if (powerLevelEvent.getContent().users[this.userId] !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users[this.userId];
-    } else if (powerLevelEvent.getContent().users_default !== undefined) {
-        this.powerLevel = powerLevelEvent.getContent().users_default;
+    if (evContent.users && evContent.users[this.userId] !== undefined) {
+        this.powerLevel = evContent.users[this.userId];
+    } else if (evContent.users_default !== undefined) {
+        this.powerLevel = evContent.users_default;
    } else {
        this.powerLevel = 0;
    }
@@ -174,6 +201,44 @@ RoomMember.prototype.getLastModifiedTime = function() {
    return this._modified;
 };

+
+RoomMember.prototype.isKicked = function() {
+    return this.membership === "leave" &&
+        this.events.member.getSender() !== this.events.member.getStateKey();
+};
+
+/**
+ * If this member was invited with the is_direct flag set, return
+ * the user that invited this member
+ * @return {string} user id of the inviter
+ */
+RoomMember.prototype.getDMInviter = function() {
+    // when not available because that room state hasn't been loaded in,
+    // we don't really know, but more likely to not be a direct chat
+    if (this.events.member) {
+        // TODO: persist the is_direct flag on the member as more member events
+        //       come in caused by displayName changes.
+
+        // the is_direct flag is set on the invite member event.
+        // This is copied on the prev_content section of the join member event
+        // when the invite is accepted.
+
+        const memberEvent = this.events.member;
+        let memberContent = memberEvent.getContent();
+        let inviteSender = memberEvent.getSender();
+
+        if (memberContent.membership === "join") {
+            memberContent = memberEvent.getPrevContent();
+            inviteSender = memberEvent.getUnsigned().prev_sender;
+        }
+
+        if (memberContent.membership === "invite" && memberContent.is_direct) {
+            return inviteSender;
+        }
+    }
+};
+
+
 /**
 * Get the avatar URL for a room member.
 * @param {string} baseUrl The base homeserver URL See
@@ -197,10 +262,12 @@ RoomMember.prototype.getAvatarUrl =
    if (allowDefault === undefined) {
        allowDefault = true;
    }
-    if (!this.events.member && !allowDefault) {
+
+    const rawUrl = this.getMxcAvatarUrl();
+
+    if (!rawUrl && !allowDefault) {
        return null;
    }
-    const rawUrl = this.events.member ? this.events.member.getContent().avatar_url : null;
    const httpUrl = ContentRepo.getHttpUriForMxc(
        baseUrl, rawUrl, width, height, resizeMethod, allowDirectLinks,
    );
@@ -213,12 +280,21 @@ RoomMember.prototype.getAvatarUrl =
    }
    return null;
 };
+/**
+ * get the mxc avatar url, either from a state event, or from a lazily loaded member
+ * @return {string} the mxc avatar url
+ */
+RoomMember.prototype.getMxcAvatarUrl = function() {
+    if(this.events.member) {
+        return this.events.member.getDirectionalContent().avatar_url;
+    } else if(this.user) {
+        return this.user.avatarUrl;
+    }
+    return null;
+};

-function calculateDisplayName(member, event, roomState) {
-    const displayName = event.getDirectionalContent().displayname;
-    const selfUserId = member.userId;
-
-    if (!displayName) {
+function calculateDisplayName(selfUserId, displayName, roomState) {
+    if (!displayName || displayName === selfUserId) {
        return selfUserId;
    }

@@ -226,11 +302,26 @@ function calculateDisplayName(member, event, roomState) {
        return displayName;
    }

-    const userIds = roomState.getUserIdsWithDisplayName(displayName);
-    const otherUsers = userIds.filter(function(u) {
-        return u !== selfUserId;
-    });
-    if (otherUsers.length > 0) {
+    // First check if the displayname is something we consider truthy
+    // after stripping it of zero width characters and padding spaces
+    const strippedDisplayName = utils.removeHiddenChars(displayName);
+    if (!strippedDisplayName) {
+        return selfUserId;
+    }
+
+    // Next check if the name contains something that look like a mxid
+    // If it does, it may be someone trying to impersonate someone else
+    // Show full mxid in this case
+    // Also show mxid if there are other people with the same displayname
+    // ignoring any zero width chars (unicode 200B-200D)
+    // if their displayname is made up of just zero width chars, show full mxid
+    let disambiguate = /@.+:.+/.test(displayName);
+    if (!disambiguate) {
+        const userIds = roomState.getUserIdsWithDisplayName(strippedDisplayName);
+        disambiguate = userIds.some((u) => u !== selfUserId);
+    }
+
+    if (disambiguate) {
        return displayName + " (" + selfUserId + ")";
    }
    return displayName;
@@ -22,18 +22,46 @@ const EventEmitter = require("events").EventEmitter;
 const utils = require("../utils");
 const RoomMember = require("./room-member");

+// possible statuses for out-of-band member loading
+const OOB_STATUS_NOTSTARTED = 1;
+const OOB_STATUS_INPROGRESS = 2;
+const OOB_STATUS_FINISHED = 3;
+
 /**
 * Construct room state.
+ *
+ * Room State represents the state of the room at a given point.
+ * It can be mutated by adding state events to it.
+ * There are two types of room member associated with a state event:
+ * normal member objects (accessed via getMember/getMembers) which mutate
+ * with the state to represent the current state of that room/user, eg.
+ * the object returned by getMember('@bob:example.com') will mutate to
+ * get a different display name if Bob later changes his display name
+ * in the room.
+ * There are also 'sentinel' members (accessed via getSentinelMember).
+ * These also represent the state of room members at the point in time
+ * represented by the RoomState object, but unlike objects from getMember,
+ * sentinel objects will always represent the room state as at the time
+ * getSentinelMember was called, so if Bob subsequently changes his display
+ * name, a room member object previously acquired with getSentinelMember
+ * will still have his old display name. Calling getSentinelMember again
+ * after the display name change will return a new RoomMember object
+ * with Bob's new display name.
+ *
 * @constructor
 * @param {?string} roomId Optional. The ID of the room which has this state.
 * If none is specified it just tracks paginationTokens, useful for notifTimelineSet
+ * @param {?object} oobMemberFlags Optional. The state of loading out of bound members.
+ * As the timeline might get reset while they are loading, this state needs to be inherited
+ * and shared when the room state is cloned for the new timeline.
+ * This should only be passed from clone.
 * @prop {Object.<string, RoomMember>} members The room member dictionary, keyed
 * on the user's ID.
 * @prop {Object.<string, Object.<string, MatrixEvent>>} events The state
 * events dictionary, keyed on the event type and then the state_key value.
 * @prop {string} paginationToken The pagination token for this state.
 */
-function RoomState(roomId) {
+function RoomState(roomId, oobMemberFlags = undefined) {
    this.roomId = roomId;
    this.members = {
        // userId: RoomMember
@@ -50,9 +78,74 @@ function RoomState(roomId) {
    this._displayNameToUserIds = {};
    this._userIdsToDisplayNames = {};
    this._tokenToInvite = {}; // 3pid invite state_key to m.room.member invite
+    this._joinedMemberCount = null; // cache of the number of joined members
+    // joined members count from summary api
+    // once set, we know the server supports the summary api
+    // and we should only trust that
+    // we could also only trust that before OOB members
+    // are loaded but doesn't seem worth the hassle atm
+    this._summaryJoinedMemberCount = null;
+    // same for invited member count
+    this._invitedMemberCount = null;
+    this._summaryInvitedMemberCount = null;
+
+    if (!oobMemberFlags) {
+        oobMemberFlags = {
+            status: OOB_STATUS_NOTSTARTED,
+        };
+    }
+    this._oobMemberFlags = oobMemberFlags;
 }
 utils.inherits(RoomState, EventEmitter);

+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+RoomState.prototype.getJoinedMemberCount = function() {
+    if (this._summaryJoinedMemberCount !== null) {
+        return this._summaryJoinedMemberCount;
+    }
+    if (this._joinedMemberCount === null) {
+        this._joinedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'join' ? count + 1 : count;
+        }, 0);
+    }
+    return this._joinedMemberCount;
+};
+
+/**
+ * Set the joined member count explicitly (like from summary part of the sync response)
+ * @param {number} count the amount of joined members
+ */
+RoomState.prototype.setJoinedMemberCount = function(count) {
+    this._summaryJoinedMemberCount = count;
+};
+/**
+ * Returns the number of invited members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite'
+ */
+RoomState.prototype.getInvitedMemberCount = function() {
+    if (this._summaryInvitedMemberCount !== null) {
+        return this._summaryInvitedMemberCount;
+    }
+    if (this._invitedMemberCount === null) {
+        this._invitedMemberCount = this.getMembers().reduce((count, m) => {
+            return m.membership === 'invite' ? count + 1 : count;
+        }, 0);
+    }
+    return this._invitedMemberCount;
+};
+
+/**
+ * Set the amount of invited members in this room
+ * @param {number} count the amount of invited members
+ */
+RoomState.prototype.setInvitedMemberCount = function(count) {
+    this._summaryInvitedMemberCount = count;
+};
+
 /**
 * Get all RoomMembers in this room.
 * @return {Array<RoomMember>} A list of RoomMembers.
@@ -80,7 +173,18 @@ RoomState.prototype.getMember = function(userId) {
 * @return {RoomMember} The member or null if they do not exist.
 */
 RoomState.prototype.getSentinelMember = function(userId) {
-    return this._sentinels[userId] || null;
+    if (!userId) return null;
+    let sentinel = this._sentinels[userId];
+
+    if (sentinel === undefined) {
+        sentinel = new RoomMember(this.roomId, userId);
+        const member = this.members[userId];
+        if (member) {
+            sentinel.setMembershipEvent(member.events.member, this);
+        }
+        this._sentinels[userId] = sentinel;
+    }
+    return sentinel;
 };

 /**
@@ -104,6 +208,67 @@ RoomState.prototype.getStateEvents = function(eventType, stateKey) {
    return event ? event : null;
 };

+/**
+ * Creates a copy of this room state so that mutations to either won't affect the other.
+ * @return {RoomState} the copy of the room state
+ */
+RoomState.prototype.clone = function() {
+    const copy = new RoomState(this.roomId, this._oobMemberFlags);
+
+    // Ugly hack: because setStateEvents will mark
+    // members as susperseding future out of bound members
+    // if loading is in progress (through _oobMemberFlags)
+    // since these are not new members, we're merely copying them
+    // set the status to not started
+    // after copying, we set back the status
+    const status = this._oobMemberFlags.status;
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+
+    Object.values(this.events).forEach((eventsByStateKey) => {
+        const eventsForType = Object.values(eventsByStateKey);
+        copy.setStateEvents(eventsForType);
+    });
+
+    // Ugly hack: see above
+    this._oobMemberFlags.status = status;
+
+    if (this._summaryInvitedMemberCount !== null) {
+        copy.setInvitedMemberCount(this.getInvitedMemberCount());
+    }
+    if (this._summaryJoinedMemberCount !== null) {
+        copy.setJoinedMemberCount(this.getJoinedMemberCount());
+    }
+
+    // copy out of band flags if needed
+    if (this._oobMemberFlags.status == OOB_STATUS_FINISHED) {
+        // copy markOutOfBand flags
+        this.getMembers().forEach((member) => {
+            if (member.isOutOfBand()) {
+                const copyMember = copy.getMember(member.userId);
+                copyMember.markOutOfBand();
+            }
+        });
+    }
+
+    return copy;
+};
+
+/**
+ * Add previously unknown state events.
+ * When lazy loading members while back-paginating,
+ * the relevant room state for the timeline chunk at the end
+ * of the chunk can be set with this method.
+ * @param {MatrixEvent[]} events state events to prepend
+ */
+RoomState.prototype.setUnknownStateEvents = function(events) {
+    const unknownStateEvents = events.filter((event) => {
+        return this.events[event.getType()] === undefined ||
+            this.events[event.getType()][event.getStateKey()] === undefined;
+    });
+
+    this.setStateEvents(unknownStateEvents);
+};
+
 /**
 * Add an array of one or more state MatrixEvents, overwriting
 * any existing state with the same {type, stateKey} tuple. Will fire
@@ -127,10 +292,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
            return;
        }

-        if (self.events[event.getType()] === undefined) {
-            self.events[event.getType()] = {};
-        }
-        self.events[event.getType()][event.getStateKey()] = event;
+        self._setStateEvent(event);
        if (event.getType() === "m.room.member") {
            _updateDisplayNameCache(
                self, event.getStateKey(), event.getContent().displayname,
@@ -168,28 +330,10 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                    event.getPrevContent().displayname;
            }

-            let member = self.members[userId];
-            if (!member) {
-                member = new RoomMember(event.getRoomId(), userId);
-                self.emit("RoomState.newMember", event, self, member);
-            }
-            // Add a new sentinel for this change. We apply the same
-            // operations to both sentinel and member rather than deep copying
-            // so we don't make assumptions about the properties of RoomMember
-            // (e.g. and manage to break it because deep copying doesn't do
-            // everything).
-            const sentinel = new RoomMember(event.getRoomId(), userId);
-            utils.forEach([member, sentinel], function(roomMember) {
-                roomMember.setMembershipEvent(event, self);
-                // this member may have a power level already, so set it.
-                const pwrLvlEvent = self.getStateEvents("m.room.power_levels", "");
-                if (pwrLvlEvent) {
-                    roomMember.setPowerLevelEvent(pwrLvlEvent);
-                }
-            });
+            const member = self._getOrCreateMember(userId, event);
+            member.setMembershipEvent(event, self);

-            self._sentinels[userId] = sentinel;
-            self.members[userId] = member;
+            self._updateMember(member);
            self.emit("RoomState.members", event, self, member);
        } else if (event.getType() === "m.room.power_levels") {
            const members = utils.values(self.members);
@@ -197,10 +341,147 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
                member.setPowerLevelEvent(event);
                self.emit("RoomState.members", event, self, member);
            });
+
+            // assume all our sentinels are now out-of-date
+            self._sentinels = {};
        }
    });
 };

+/**
+ * Looks up a member by the given userId, and if it doesn't exist,
+ * create it and emit the `RoomState.newMember` event.
+ * This method makes sure the member is added to the members dictionary
+ * before emitting, as this is done from setStateEvents and _setOutOfBandMember.
+ * @param {string} userId the id of the user to look up
+ * @param {MatrixEvent} event the membership event for the (new) member. Used to emit.
+ * @fires module:client~MatrixClient#event:"RoomState.newMember"
+ * @returns {RoomMember} the member, existing or newly created.
+ */
+RoomState.prototype._getOrCreateMember = function(userId, event) {
+    let member = this.members[userId];
+    if (!member) {
+        member = new RoomMember(this.roomId, userId);
+        // add member to members before emitting any events,
+        // as event handlers often lookup the member
+        this.members[userId] = member;
+        this.emit("RoomState.newMember", event, this, member);
+    }
+    return member;
+};
+
+RoomState.prototype._setStateEvent = function(event) {
+    if (this.events[event.getType()] === undefined) {
+        this.events[event.getType()] = {};
+    }
+    this.events[event.getType()][event.getStateKey()] = event;
+};
+
+RoomState.prototype._updateMember = function(member) {
+    // this member may have a power level already, so set it.
+    const pwrLvlEvent = this.getStateEvents("m.room.power_levels", "");
+    if (pwrLvlEvent) {
+        member.setPowerLevelEvent(pwrLvlEvent);
+    }
+
+    // blow away the sentinel which is now outdated
+    delete this._sentinels[member.userId];
+
+    this.members[member.userId] = member;
+    this._joinedMemberCount = null;
+    this._invitedMemberCount = null;
+};
+
+/**
+ * Get the out-of-band members loading state, whether loading is needed or not.
+ * Note that loading might be in progress and hence isn't needed.
+ * @return {bool} whether or not the members of this room need to be loaded
+ */
+RoomState.prototype.needsOutOfBandMembers = function() {
+    return this._oobMemberFlags.status === OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Mark this room state as waiting for out-of-band members,
+ * ensuring it doesn't ask for them to be requested again
+ * through needsOutOfBandMembers
+ */
+RoomState.prototype.markOutOfBandMembersStarted = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_NOTSTARTED) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_INPROGRESS;
+};
+
+/**
+ * Mark this room state as having failed to fetch out-of-band members
+ */
+RoomState.prototype.markOutOfBandMembersFailed = function() {
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Clears the loaded out-of-band members
+ */
+RoomState.prototype.clearOutOfBandMembers = function() {
+    let count = 0;
+    Object.keys(this.members).forEach((userId) => {
+        const member = this.members[userId];
+        if (member.isOutOfBand()) {
+            ++count;
+            delete this.members[userId];
+        }
+    });
+    console.log(`LL: RoomState removed ${count} members...`);
+    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;
+};
+
+/**
+ * Sets the loaded out-of-band members.
+ * @param {MatrixEvent[]} stateEvents array of membership state events
+ */
+RoomState.prototype.setOutOfBandMembers = function(stateEvents) {
+    console.log(`LL: RoomState about to set ${stateEvents.length} OOB members ...`);
+    if (this._oobMemberFlags.status !== OOB_STATUS_INPROGRESS) {
+        return;
+    }
+    console.log(`LL: RoomState put in OOB_STATUS_FINISHED state ...`);
+    this._oobMemberFlags.status = OOB_STATUS_FINISHED;
+    stateEvents.forEach((e) => this._setOutOfBandMember(e));
+};
+
+/**
+ * Sets a single out of band member, used by both setOutOfBandMembers and clone
+ * @param {MatrixEvent} stateEvent membership state event
+ */
+RoomState.prototype._setOutOfBandMember = function(stateEvent) {
+    if (stateEvent.getType() !== 'm.room.member') {
+        return;
+    }
+    const userId = stateEvent.getStateKey();
+    const existingMember = this.getMember(userId);
+    // never replace members received as part of the sync
+    if (existingMember && !existingMember.isOutOfBand()) {
+        return;
+    }
+
+    const member = this._getOrCreateMember(userId, stateEvent);
+    member.setMembershipEvent(stateEvent, this);
+    // needed to know which members need to be stored seperately
+    // as they are not part of the sync accumulator
+    // this is cleared by setMembershipEvent so when it's updated through /sync
+    member.markOutOfBand();
+
+    _updateDisplayNameCache(this, member.userId, member.name);
+
+    this._setStateEvent(stateEvent);
+    this._updateMember(member);
+    this.emit("RoomState.members", stateEvent, this, member);
+};
+
 /**
 * Set the current typing event for this room.
 * @param {MatrixEvent} event The typing event
@@ -258,7 +539,11 @@ RoomState.prototype.maySendRedactionForEvent = function(mxEvent, userId) {
    if (!member || member.membership === 'leave') return false;

    if (mxEvent.status || mxEvent.isRedacted()) return false;
-    if (mxEvent.getSender() === userId) return true;
+
+    // The user may have been the sender, but they can't redact their own message
+    // if redactions are blocked.
+    const canRedact = this.maySendEvent("m.room.redaction", userId);
+    if (mxEvent.getSender() === userId) return canRedact;

    return this._hasSufficientPowerLevelFor('redact', member.powerLevel);
 };
@@ -278,7 +563,7 @@ RoomState.prototype._hasSufficientPowerLevelFor = function(action, powerLevel) {
    }

    let requiredLevel = 50;
-    if (powerLevels[action] !== undefined) {
+    if (utils.isNumber(powerLevels[action])) {
        requiredLevel = powerLevels[action];
    }

@@ -350,11 +635,6 @@ RoomState.prototype.maySendStateEvent = function(stateEventType, userId) {
 *                        according to the room's state.
 */
 RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
-    const member = this.getMember(userId);
-    if (!member || member.membership == 'leave') {
-        return false;
-    }
-
    const power_levels_event = this.getStateEvents('m.room.power_levels', '');

    let power_levels;
@@ -362,25 +642,63 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {

    let state_default = 0;
    let events_default = 0;
+    let powerLevel = 0;
    if (power_levels_event) {
        power_levels = power_levels_event.getContent();
        events_levels = power_levels.events || {};

-        if (power_levels.state_default !== undefined) {
+        if (Number.isFinite(power_levels.state_default)) {
            state_default = power_levels.state_default;
        } else {
            state_default = 50;
        }
-        if (power_levels.events_default !== undefined) {
+
+        const userPowerLevel = power_levels.users && power_levels.users[userId];
+        if (Number.isFinite(userPowerLevel)) {
+            powerLevel = userPowerLevel;
+        } else if(Number.isFinite(power_levels.users_default)) {
+            powerLevel = power_levels.users_default;
+        }
+
+        if (Number.isFinite(power_levels.events_default)) {
            events_default = power_levels.events_default;
        }
    }

    let required_level = state ? state_default : events_default;
-    if (events_levels[eventType] !== undefined) {
+    if (Number.isFinite(events_levels[eventType])) {
        required_level = events_levels[eventType];
    }
-    return member.powerLevel >= required_level;
+    return powerLevel >= required_level;
+};
+
+/**
+ * Returns true if the given user ID has permission to trigger notification
+ * of type `notifLevelKey`
+ * @param {string} notifLevelKey The level of notification to test (eg. 'room')
+ * @param {string} userId The user ID of the user to test permission for
+ * @return {boolean} true if the given user ID has permission to trigger a
+ *                        notification of this type.
+ */
+RoomState.prototype.mayTriggerNotifOfType = function(notifLevelKey, userId) {
+    const member = this.getMember(userId);
+    if (!member) {
+        return false;
+    }
+
+    const powerLevelsEvent = this.getStateEvents('m.room.power_levels', '');
+
+    let notifLevel = 50;
+    if (
+        powerLevelsEvent &&
+        powerLevelsEvent.getContent() &&
+        powerLevelsEvent.getContent().notifications &&
+        utils.isNumber(powerLevelsEvent.getContent().notifications[notifLevelKey])
+    ) {
+        notifLevel = powerLevelsEvent.getContent().notifications[notifLevelKey];
+    }
+
+    return member.powerLevel >= notifLevel;
 };

 /**
@@ -414,22 +732,26 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
        // We clobber the user_id > name lookup but the name -> [user_id] lookup
        // means we need to remove that user ID from that array rather than nuking
        // the lot.
-        const existingUserIds = roomState._displayNameToUserIds[oldName] || [];
-        for (let i = 0; i < existingUserIds.length; i++) {
-            if (existingUserIds[i] === userId) {
-                // remove this user ID from this array
-                existingUserIds.splice(i, 1);
-                i--;
-            }
+        const strippedOldName = utils.removeHiddenChars(oldName);
+
+        const existingUserIds = roomState._displayNameToUserIds[strippedOldName];
+        if (existingUserIds) {
+            // remove this user ID from this array
+            const filteredUserIDs = existingUserIds.filter((id) => id !== userId);
+            roomState._displayNameToUserIds[strippedOldName] = filteredUserIDs;
        }
-        roomState._displayNameToUserIds[oldName] = existingUserIds;
    }

    roomState._userIdsToDisplayNames[userId] = displayName;
-    if (!roomState._displayNameToUserIds[displayName]) {
-        roomState._displayNameToUserIds[displayName] = [];
+
+    const strippedDisplayname = displayName && utils.removeHiddenChars(displayName);
+    // an empty stripped displayname (undefined/'') will be set to MXID in room-member.js
+    if (strippedDisplayname) {
+        if (!roomState._displayNameToUserIds[strippedDisplayname]) {
+            roomState._displayNameToUserIds[strippedDisplayname] = [];
+        }
+        roomState._displayNameToUserIds[strippedDisplayname].push(userId);
    }
-    roomState._displayNameToUserIds[displayName].push(userId);
 }

 /**
@@ -459,7 +781,8 @@ function _updateDisplayNameCache(roomState, userId, displayName) {

 /**
 * Fires whenever a member is added to the members dictionary. The RoomMember
- * will not be fully populated yet (e.g. no membership state).
+ * will not be fully populated yet (e.g. no membership state) but will already
+ * be available in the members dictionary.
 * @event module:client~MatrixClient#"RoomState.newMember"
 * @param {MatrixEvent} event The matrix event which caused this event to fire.
 * @param {RoomState} state The room state whose RoomState.members dictionary
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -21,6 +22,7 @@ const EventEmitter = require("events").EventEmitter;

 const EventStatus = require("./event").EventStatus;
 const RoomSummary = require("./room-summary");
+const RoomMember = require("./room-member");
 const MatrixEvent = require("./event").MatrixEvent;
 const utils = require("../utils");
 const ContentRepo = require("../content-repo");
@@ -29,6 +31,8 @@ const EventTimelineSet = require("./event-timeline-set");

 import ReEmitter from '../ReEmitter';

+const LATEST_ROOM_VERSION = '1';
+
 function synthesizeReceipt(userId, event, receiptType) {
    // console.log("synthesizing receipt for "+event.getId());
    // This is really ugly because JS has no way to express an object literal
@@ -68,6 +72,8 @@ function synthesizeReceipt(userId, event, receiptType) {
 * @constructor
 * @alias module:models/room
 * @param {string} roomId Required. The ID of this room.
+ * @param {MatrixClient} client Required. The client, used to lazy load members.
+ * @param {string} myUserId Required. The ID of the syncing user.
 * @param {Object=} opts Configuration options
 * @param {*} opts.storageToken Optional. The token which a data store can use
 * to remember the state of the room. What this means is dependent on the store
@@ -94,15 +100,15 @@ function synthesizeReceipt(userId, event, receiptType) {
 * event type and the values are the events.
 * @prop {RoomState} oldState The state of the room at the time of the oldest
 * event in the live timeline. Present for backwards compatibility -
- * prefer getLiveTimeline().getState(true).
+ * prefer getLiveTimeline().getState(EventTimeline.BACKWARDS).
 * @prop {RoomState} currentState The state of the room at the time of the
 * newest event in the timeline. Present for backwards compatibility -
- * prefer getLiveTimeline().getState(false).
+ * prefer getLiveTimeline().getState(EventTimeline.FORWARDS).
 * @prop {RoomSummary} summary The room summary.
 * @prop {*} storageToken A token which a data store can use to remember
 * the state of the room.
 */
-function Room(roomId, opts) {
+function Room(roomId, client, myUserId, opts) {
    opts = opts || {};
    opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";

@@ -115,6 +121,7 @@ function Room(roomId, opts) {
        );
    }

+    this.myUserId = myUserId;
    this.roomId = roomId;
    this.name = roomId;
    this.tags = {
@@ -169,10 +176,58 @@ function Room(roomId, opts) {
        this._pendingEventList = [];
    }

-    this._blacklistUnverifiedDevices = false; // read by megolm
+    // read by megolm; boolean value - null indicates "use global value"
+    this._blacklistUnverifiedDevices = null;
+    this._selfMembership = null;
+    this._summaryHeroes = null;
+    // awaited by getEncryptionTargetMembers while room members are loading
+
+    this._client = client;
+    if (!this._opts.lazyLoadMembers) {
+        this._membersPromise = Promise.resolve();
+    } else {
+        this._membersPromise = null;
+    }
 }
+
 utils.inherits(Room, EventEmitter);

+/**
+ * Gets the version of the room
+ * @returns {string} The version of the room, or null if it could not be determined
+ */
+Room.prototype.getVersion = function() {
+    const createEvent = this.currentState.getStateEvents("m.room.create", "");
+    if (!createEvent) {
+        console.warn("Room " + this.room_id + " does not have an m.room.create event");
+        return '1';
+    }
+    const ver = createEvent.getContent()['room_version'];
+    if (ver === undefined) return '1';
+    return ver;
+};
+
+/**
+ * Determines whether this room needs to be upgraded to a new version
+ * @returns {string?} What version the room should be upgraded to, or null if
+ *     the room does not require upgrading at this time.
+ */
+Room.prototype.shouldUpgradeToVersion = function() {
+    // This almost certainly won't be the way this actually works - this
+    // is essentially a stub method.
+    if (this.getVersion() === LATEST_ROOM_VERSION) return null;
+    return LATEST_ROOM_VERSION;
+};
+
+/**
+ * Determines whether the given user is permitted to perform a room upgrade
+ * @param {String} userId The ID of the user to test against
+ * @returns {bool} True if the given user is permitted to upgrade the room
+ */
+Room.prototype.userMayUpgradeRoom = function(userId) {
+    return this.currentState.maySendStateEvent("m.room.tombstone", userId);
+};
+
 /**
 * Get the list of pending sent events for this room
 *
@@ -200,6 +255,232 @@ Room.prototype.getLiveTimeline = function() {
    return this.getUnfilteredTimelineSet().getLiveTimeline();
 };

+/**
+ * @param {string} myUserId the user id for the logged in member
+ * @return {string} the membership type (join | leave | invite) for the logged in user
+ */
+Room.prototype.getMyMembership = function() {
+    return this._selfMembership;
+};
+
+/**
+ * If this room is a DM we're invited to,
+ * try to find out who invited us
+ * @return {string} user id of the inviter
+ */
+Room.prototype.getDMInviter = function() {
+    if (this.myUserId) {
+        const me = this.getMember(this.myUserId);
+        if (me) {
+            return me.getDMInviter();
+        }
+    }
+    if (this._selfMembership === "invite") {
+        // fall back to summary information
+        const memberCount = this.getInvitedAndJoinedMemberCount();
+        if (memberCount == 2 && this._summaryHeroes.length) {
+            return this._summaryHeroes[0];
+        }
+    }
+};
+
+/**
+ * Assuming this room is a DM room, tries to guess with which user.
+ * @return {string} user id of the other member (could be syncing user)
+ */
+Room.prototype.guessDMUserId = function() {
+    const me = this.getMember(this.myUserId);
+    if (me) {
+        const inviterId = me.getDMInviter();
+        if (inviterId) {
+            return inviterId;
+        }
+    }
+    // remember, we're assuming this room is a DM,
+    // so returning the first member we find should be fine
+    const hasHeroes = Array.isArray(this._summaryHeroes) &&
+        this._summaryHeroes.length;
+    if (hasHeroes) {
+        return this._summaryHeroes[0];
+    }
+    const members = this.currentState.getMembers();
+    const anyMember = members.find((m) => m.userId !== this.myUserId);
+    if (anyMember) {
+        return anyMember.userId;
+    }
+    // it really seems like I'm the only user in the room
+    // so I probably created a room with just me in it
+    // and marked it as a DM. Ok then
+    return this.myUserId;
+};
+
+Room.prototype.getAvatarFallbackMember = function() {
+    const memberCount = this.getInvitedAndJoinedMemberCount();
+    if (memberCount > 2) {
+        return;
+    }
+    const hasHeroes = Array.isArray(this._summaryHeroes) &&
+        this._summaryHeroes.length;
+    if (hasHeroes) {
+        const availableMember = this._summaryHeroes.map((userId) => {
+            return this.getMember(userId);
+        }).find((member) => !!member);
+        if (availableMember) {
+            return availableMember;
+        }
+    }
+    const members = this.currentState.getMembers();
+    // could be different than memberCount
+    // as this includes left members
+    if (members.length <= 2) {
+        const availableMember = members.find((m) => {
+            return m.userId !== this.myUserId;
+        });
+        if (availableMember) {
+            return availableMember;
+        }
+    }
+    // if all else fails, try falling back to a user,
+    // and create a one-off member for it
+    if (hasHeroes) {
+        const availableUser = this._summaryHeroes.map((userId) => {
+            return this._client.getUser(userId);
+        }).find((user) => !!user);
+        if (availableUser) {
+            const member = new RoomMember(
+                this.roomId, availableUser.userId);
+            member.user = availableUser;
+            return member;
+        }
+    }
+};
+
+/**
+ * Sets the membership this room was received as during sync
+ * @param {string} membership join | leave | invite
+ */
+Room.prototype.updateMyMembership = function(membership) {
+    const prevMembership = this._selfMembership;
+    this._selfMembership = membership;
+    if (prevMembership !== membership) {
+        if (membership === "leave") {
+            this._cleanupAfterLeaving();
+        }
+        this.emit("Room.myMembership", this, membership, prevMembership);
+    }
+};
+
+Room.prototype._loadMembersFromServer = async function() {
+    const lastSyncToken = this._client.store.getSyncToken();
+    const queryString = utils.encodeParams({
+        not_membership: "leave",
+        at: lastSyncToken,
+    });
+    const path = utils.encodeUri("/rooms/$roomId/members?" + queryString,
+        {$roomId: this.roomId});
+    const http = this._client._http;
+    const response = await http.authedRequest(undefined, "GET", path);
+    return response.chunk;
+};
+
+
+Room.prototype._loadMembers = async function() {
+    // were the members loaded from the server?
+    let fromServer = false;
+    let rawMembersEvents =
+        await this._client.store.getOutOfBandMembers(this.roomId);
+    if (rawMembersEvents === null) {
+        fromServer = true;
+        rawMembersEvents = await this._loadMembersFromServer();
+        console.log(`LL: got ${rawMembersEvents.length} ` +
+            `members from server for room ${this.roomId}`);
+    }
+    const memberEvents = rawMembersEvents.map(this._client.getEventMapper());
+    return {memberEvents, fromServer};
+};
+
+/**
+ * Preloads the member list in case lazy loading
+ * of memberships is in use. Can be called multiple times,
+ * it will only preload once.
+ * @return {Promise} when preloading is done and
+ * accessing the members on the room will take
+ * all members in the room into account
+ */
+Room.prototype.loadMembersIfNeeded = function() {
+    if (this._membersPromise) {
+        return this._membersPromise;
+    }
+
+    // mark the state so that incoming messages while
+    // the request is in flight get marked as superseding
+    // the OOB members
+    this.currentState.markOutOfBandMembersStarted();
+
+    const inMemoryUpdate = this._loadMembers().then((result) => {
+        this.currentState.setOutOfBandMembers(result.memberEvents);
+        // now the members are loaded, start to track the e2e devices if needed
+        if (this._client.isRoomEncrypted(this.roomId)) {
+            this._client._crypto.trackRoomDevices(this.roomId);
+        }
+        return result.fromServer;
+    }).catch((err) => {
+        // allow retries on fail
+        this._membersPromise = null;
+        this.currentState.markOutOfBandMembersFailed();
+        throw err;
+    });
+    // update members in storage, but don't wait for it
+    inMemoryUpdate.then((fromServer) => {
+        if (fromServer) {
+            const oobMembers = this.currentState.getMembers()
+                .filter((m) => m.isOutOfBand())
+                .map((m) => m.events.member.event);
+            console.log(`LL: telling store to write ${oobMembers.length}`
+                + ` members for room ${this.roomId}`);
+            const store = this._client.store;
+            return store.setOutOfBandMembers(this.roomId, oobMembers)
+                // swallow any IDB error as we don't want to fail
+                // because of this
+                .catch((err) => {
+                    console.log("LL: storing OOB room members failed, oh well",
+                        err);
+                });
+        }
+    }).catch((err) => {
+        // as this is not awaited anywhere,
+        // at least show the error in the console
+        console.error(err);
+    });
+
+    this._membersPromise = inMemoryUpdate;
+
+    return this._membersPromise;
+};
+
+/**
+ * Removes the lazily loaded members from storage if needed
+ */
+Room.prototype.clearLoadedMembersIfNeeded = async function() {
+    if (this._opts.lazyLoadMembers && this._membersPromise) {
+        await this.loadMembersIfNeeded();
+        await this._client.store.clearOutOfBandMembers(this.roomId);
+        this.currentState.clearOutOfBandMembers();
+        this._membersPromise = null;
+    }
+};
+
+/**
+ * called when sync receives this room in the leave section
+ * to do cleanup after leaving a room. Possibly called multiple times.
+ */
+Room.prototype._cleanupAfterLeaving = function() {
+    this.clearLoadedMembersIfNeeded().catch((err) => {
+        console.error(`error after clearing loaded members from ` +
+            `room ${this.roomId} after leaving`);
+        console.dir(err);
+    });
+};

 /**
 * Reset the live timeline of all timelineSets, and start new ones.
@@ -305,10 +586,30 @@ Room.prototype.setUnreadNotificationCount = function(type, count) {
    this._notificationCounts[type] = count;
 };

+Room.prototype.setSummary = function(summary) {
+    const heroes = summary["m.heroes"];
+    const joinedCount = summary["m.joined_member_count"];
+    const invitedCount = summary["m.invited_member_count"];
+    if (Number.isInteger(joinedCount)) {
+        this.currentState.setJoinedMemberCount(joinedCount);
+    }
+    if (Number.isInteger(invitedCount)) {
+        this.currentState.setInvitedMemberCount(invitedCount);
+    }
+    if (Array.isArray(heroes)) {
+        // be cautious about trusting server values,
+        // and make sure heroes doesn't contain our own id
+        // just to be sure
+        this._summaryHeroes = heroes.filter((userId) => {
+            return userId !== this.myUserId;
+        });
+    }
+};
+
 /**
 * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @param {boolean} value if true, blacklist unverified devices.
+ * @param {Boolean} value true to blacklist unverified devices, null
+ * to use the global value for this room.
 */
 Room.prototype.setBlacklistUnverifiedDevices = function(value) {
    this._blacklistUnverifiedDevices = value;
@@ -316,8 +617,8 @@ Room.prototype.setBlacklistUnverifiedDevices = function(value) {

 /**
 * Whether to send encrypted messages to devices within this room.
- * Will be ignored if MatrixClient's blacklistUnverifiedDevices setting is true.
- * @return {boolean} true if blacklisting unverified devices.
+ * @return {Boolean} true if blacklisting unverified devices, null
+ * if the global value should be used for this room.
 */
 Room.prototype.getBlacklistUnverifiedDevices = function() {
    return this._blacklistUnverifiedDevices;
@@ -429,11 +730,7 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
 * @return {RoomMember} The member or <code>null</code>.
 */
 Room.prototype.getMember = function(userId) {
-    const member = this.currentState.members[userId];
-    if (!member) {
-        return null;
-    }
-    return member;
+    return this.currentState.getMember(userId);
 };

 /**
@@ -444,6 +741,33 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
    return this.getMembersWithMembership("join");
 };

+/**
+ * Returns the number of joined members in this room
+ * This method caches the result.
+ * This is a wrapper around the method of the same name in roomState, returning
+ * its result for the room's current state.
+ * @return {integer} The number of members in this room whose membership is 'join'
+ */
+Room.prototype.getJoinedMemberCount = function() {
+    return this.currentState.getJoinedMemberCount();
+};
+
+/**
+ * Returns the number of invited members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite'
+ */
+Room.prototype.getInvitedMemberCount = function() {
+    return this.currentState.getInvitedMemberCount();
+};
+
+/**
+ * Returns the number of invited + joined members in this room
+ * @return {integer} The number of members in this room whose membership is 'invite' or 'join'
+ */
+Room.prototype.getInvitedAndJoinedMemberCount = function() {
+    return this.getInvitedMemberCount() + this.getJoinedMemberCount();
+};
+
 /**
 * Get a list of members with given membership state.
 * @param {string} membership The membership state.
@@ -455,6 +779,29 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline,
    });
 };

+ /**
+  * Get a list of members we should be encrypting for in this room
+  * @return {Promise<RoomMember[]>} A list of members who
+  * we should encrypt messages for in this room.
+  */
+ Room.prototype.getEncryptionTargetMembers = async function() {
+    await this.loadMembersIfNeeded();
+    let members = this.getMembersWithMembership("join");
+    if (this.shouldEncryptForInvitedMembers()) {
+        members = members.concat(this.getMembersWithMembership("invite"));
+    }
+    return members;
+ };
+
+ /**
+  * Determine whether we should encrypt messages for invited users in this room
+  * @return {boolean} if we should encrypt messages for invited users
+  */
+ Room.prototype.shouldEncryptForInvitedMembers = function() {
+    const ev = this.currentState.getStateEvents("m.room.history_visibility", "");
+    return (ev && ev.getContent() && ev.getContent().history_visibility !== "joined");
+ };
+
 /**
  * Get the default room name (i.e. what a given user would see if the
  * room had no m.room.name)
@@ -596,7 +943,8 @@ Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
    // synthesize and inject implicit read receipts
    // Done after adding the event because otherwise the app would get a read receipt
    // pointing to an event that wasn't yet in the timeline
-    if (event.sender) {
+    // Don't synthesize RR for m.room.redaction as this causes the RR to go missing.
+    if (event.sender && event.getType() !== "m.room.redaction") {
        this.addReceipt(synthesizeReceipt(
            event.sender.userId, event, "m.read",
        ), true);
@@ -651,6 +999,10 @@ Room.prototype.addPendingEvent = function(event, txnId) {
    this._txnToEvent[txnId] = event;

    if (this._opts.pendingEventOrdering == "detached") {
+        if (this._pendingEventList.some((e) => e.status === EventStatus.NOT_SENT)) {
+            console.warn("Setting event as NOT_SENT due to messages in the same state");
+            event.status = EventStatus.NOT_SENT;
+        }
        this._pendingEventList.push(event);
    } else {
        for (let i = 0; i < this._timelineSets.length; i++) {
@@ -907,15 +1259,14 @@ Room.prototype.removeEvent = function(eventId) {
 * Recalculate various aspects of the room, including the room name and
 * room summary. Call this any time the room's current state is modified.
 * May fire "Room.name" if the room name is updated.
- * @param {string} userId The client's user ID.
 * @fires module:client~MatrixClient#event:"Room.name"
 */
-Room.prototype.recalculate = function(userId) {
+Room.prototype.recalculate = function() {
    // set fake stripped state events if this is an invite room so logic remains
    // consistent elsewhere.
    const self = this;
    const membershipEvent = this.currentState.getStateEvents(
-        "m.room.member", userId,
+        "m.room.member", this.myUserId,
    );
    if (membershipEvent && membershipEvent.getContent().membership === "invite") {
        const strippedStateEvents = membershipEvent.event.invite_room_state || [];
@@ -931,14 +1282,14 @@ Room.prototype.recalculate = function(userId) {
                    content: strippedEvent.content,
                    event_id: "$fake" + Date.now(),
                    room_id: self.roomId,
-                    user_id: userId, // technically a lie
+                    user_id: self.myUserId, // technically a lie
                })]);
            }
        });
    }

    const oldName = this.name;
-    this.name = calculateRoomName(this, userId);
+    this.name = calculateRoomName(this, this.myUserId);
    this.summary = new RoomSummary(this.roomId, {
        title: this.name,
    });
@@ -948,7 +1299,6 @@ Room.prototype.recalculate = function(userId) {
    }
 };

-
 /**
 * Get a list of user IDs who have <b>read up to</b> the given event.
 * @param {MatrixEvent} event the event to get read receipts for.
@@ -1120,7 +1470,7 @@ Room.prototype.addTags = function(event) {
    // }

    // XXX: do we need to deep copy here?
-    this.tags = event.getContent().tags;
+    this.tags = event.getContent().tags || {};

    // XXX: we could do a deep-comparison to see if the tags have really
    // changed - but do we want to bother?
@@ -1151,6 +1501,17 @@ Room.prototype.getAccountData = function(type) {
    return this.accountData[type];
 };

+
+/**
+ * Returns wheter the syncing user has permission to send a message in the room
+ * @return {boolean} true if the user should be permitted to send
+ *                   message events into the room.
+ */
+Room.prototype.maySendMessage = function() {
+    return this.getMyMembership() === 'join' &&
+        this.currentState.maySendEvent('m.room.message', this.myUserId);
+};
+
 /**
 * This is an internal method. Calculates the name of the room from the current
 * room state.
@@ -1184,80 +1545,83 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
        return alias;
    }

-    // get members that are NOT ourselves and are actually in the room.
-    const otherMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (
-            m.userId !== userId && m.membership !== "leave" && m.membership !== "ban"
-        );
-    });
-    const allMembers = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.membership !== "leave");
-    });
-    const myMemberEventArray = utils.filter(room.currentState.getMembers(), function(m) {
-        return (m.userId == userId);
-    });
-    const myMemberEvent = (
-        (myMemberEventArray.length && myMemberEventArray[0].events) ?
-            myMemberEventArray[0].events.member.event : undefined
-    );
+    const joinedMemberCount = room.currentState.getJoinedMemberCount();
+    const invitedMemberCount = room.currentState.getInvitedMemberCount();
+    // -1 because these numbers include the syncing user
+    const inviteJoinCount = joinedMemberCount + invitedMemberCount - 1;

-    // TODO: Localisation
-    if (myMemberEvent && myMemberEvent.content.membership == "invite") {
-        if (room.currentState.getMember(myMemberEvent.sender)) {
-            // extract who invited us to the room
-            return "Invite from " + room.currentState.getMember(
-                myMemberEvent.sender,
-            ).name;
-        } else if (allMembers[0].events.member) {
-            // use the sender field from the invite event, although this only
-            // gets us the mxid
-            return "Invite from " + myMemberEvent.sender;
-        } else {
-            return "Room Invite";
-        }
+    // get members that are NOT ourselves and are actually in the room.
+    let otherNames = null;
+    if (room._summaryHeroes) {
+        // if we have a summary, the member state events
+        // should be in the room state
+        otherNames = room._summaryHeroes.map((userId) => {
+            const member = room.getMember(userId);
+            return member ? member.name : userId;
+        });
+    } else {
+        let otherMembers = room.currentState.getMembers().filter((m) => {
+            return m.userId !== userId &&
+                (m.membership === "invite" || m.membership === "join");
+        });
+        // make sure members have stable order
+        otherMembers.sort((a, b) => a.userId.localeCompare(b.userId));
+        // only 5 first members, immitate _summaryHeroes
+        otherMembers = otherMembers.slice(0, 5);
+        otherNames = otherMembers.map((m) => m.name);
    }

+    if (inviteJoinCount) {
+        return memberNamesToRoomName(otherNames, inviteJoinCount);
+    }

-    if (otherMembers.length === 0) {
-        if (allMembers.length === 1) {
-            // self-chat, peeked room with 1 participant,
-            // or inbound invite, or outbound 3PID invite.
-            if (allMembers[0].userId === userId) {
-                const thirdPartyInvites =
-                    room.currentState.getStateEvents("m.room.third_party_invite");
-                if (thirdPartyInvites && thirdPartyInvites.length > 0) {
-                    let name = "Inviting " +
-                               thirdPartyInvites[0].getContent().display_name;
-                    if (thirdPartyInvites.length > 1) {
-                        if (thirdPartyInvites.length == 2) {
-                            name += " and " +
-                                    thirdPartyInvites[1].getContent().display_name;
-                        } else {
-                            name += " and " +
-                                    thirdPartyInvites.length + " others";
-                        }
-                    }
-                    return name;
-                } else {
-                    return "Empty room";
-                }
-            } else {
-                return allMembers[0].name;
-            }
-        } else {
-            // there really isn't anyone in this room...
-            return "Empty room";
+    const myMembership = room.getMyMembership();
+    // if I have created a room and invited people throuh
+    // 3rd party invites
+    if (myMembership == 'join') {
+        const thirdPartyInvites =
+            room.currentState.getStateEvents("m.room.third_party_invite");
+
+        if (thirdPartyInvites && thirdPartyInvites.length) {
+            const thirdPartyNames = thirdPartyInvites.map((i) => {
+                return i.getContent().display_name;
+            });
+
+            return `Inviting ${memberNamesToRoomName(thirdPartyNames)}`;
        }
-    } else if (otherMembers.length === 1) {
-        return otherMembers[0].name;
-    } else if (otherMembers.length === 2) {
-        return (
-            otherMembers[0].name + " and " + otherMembers[1].name
-        );
+    }
+    // let's try to figure out who was here before
+    let leftNames = otherNames;
+    // if we didn't have heroes, try finding them in the room state
+    if(!leftNames.length) {
+        leftNames = room.currentState.getMembers().filter((m) => {
+            return m.userId !== userId &&
+                m.membership !== "invite" &&
+                m.membership !== "join";
+        }).map((m) => m.name);
+    }
+    if(leftNames.length) {
+        return `Empty room (was ${memberNamesToRoomName(leftNames)})`;
    } else {
-        return (
-            otherMembers[0].name + " and " + (otherMembers.length - 1) + " others"
-        );
+        return "Empty room";
+    }
+}
+
+function memberNamesToRoomName(names, count = (names.length + 1)) {
+    const countWithoutMe = count - 1;
+    if (!names.length) {
+       return count <= 1 ? "Empty room" : null;
+    } else if (names.length === 1 && countWithoutMe <= 1) {
+        return names[0];
+    } else if (names.length === 2 && countWithoutMe <= 2) {
+        return `${names[0]} and ${names[1]}`;
+    } else {
+        const plural = countWithoutMe > 1;
+        if (plural) {
+            return `${names[0]} and ${countWithoutMe} others`;
+        } else {
+            return `${names[0]} and 1 other`;
+        }
    }
 }

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,6 +18,8 @@ limitations under the License.
 * @module pushprocessor
 */

+const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];
+
 /**
 * Construct a Push Processor.
 * @constructor
@@ -27,12 +30,15 @@ function PushProcessor(client) {
        return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
    };

-    const matchingRuleFromKindSet = function(ev, kindset, device) {
-        const rulekinds_in_order = ['override', 'content', 'room', 'sender', 'underride'];
+    const cachedGlobToRegex = {
+        // $glob: RegExp,
+    };
+
+    const matchingRuleFromKindSet = (ev, kindset, device) => {
        for (let ruleKindIndex = 0;
-                ruleKindIndex < rulekinds_in_order.length;
+                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                ++ruleKindIndex) {
-            const kind = rulekinds_in_order[ruleKindIndex];
+            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
            const ruleset = kindset[kind];

            for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
@@ -46,7 +52,7 @@ function PushProcessor(client) {
                    continue;
                }

-                if (ruleMatchesEvent(rawrule, ev)) {
+                if (this.ruleMatchesEvent(rawrule, ev)) {
                    rule.kind = kind;
                    return rule;
                }
@@ -73,7 +79,7 @@ function PushProcessor(client) {
                rawrule.conditions.push({
                    'kind': 'event_match',
                    'key': 'room_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                });
                break;
            case 'sender':
@@ -83,7 +89,7 @@ function PushProcessor(client) {
                rawrule.conditions.push({
                    'kind': 'event_match',
                    'key': 'user_id',
-                    'pattern': tprule.rule_id,
+                    'value': tprule.rule_id,
                });
                break;
            case 'content':
@@ -106,27 +112,38 @@ function PushProcessor(client) {
        return rawrule;
    };

-    const ruleMatchesEvent = function(rule, ev) {
-        let ret = true;
-        for (let i = 0; i < rule.conditions.length; ++i) {
-            const cond = rule.conditions[i];
-            ret &= eventFulfillsCondition(cond, ev);
-        }
-        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
-        return ret;
-    };
-
    const eventFulfillsCondition = function(cond, ev) {
        const condition_functions = {
            "event_match": eventFulfillsEventMatchCondition,
            "device": eventFulfillsDeviceCondition,
            "contains_display_name": eventFulfillsDisplayNameCondition,
            "room_member_count": eventFulfillsRoomMemberCountCondition,
+            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
        };
        if (condition_functions[cond.kind]) {
            return condition_functions[cond.kind](cond, ev);
        }
-        return true;
+        // unknown conditions: we previously matched all unknown conditions,
+        // but given that rules can be added to the base rules on a server,
+        // it's probably better to not match unknown conditions.
+        return false;
+    };
+
+    const eventFulfillsSenderNotifPermCondition = function(cond, ev) {
+        const notifLevelKey = cond['key'];
+        if (!notifLevelKey) {
+            return false;
+        }
+
+        const room = client.getRoom(ev.getRoomId());
+        if (!room || !room.currentState) {
+            return false;
+        }
+
+        // Note that this should not be the current state of the room but the state at
+        // the point the event is in the DAG. Unfortunately the js-sdk does not store
+        // this.
+        return room.currentState.mayTriggerNotifOfType(notifLevelKey, ev.getSender());
    };

    const eventFulfillsRoomMemberCountCondition = function(cond, ev) {
@@ -139,9 +156,7 @@ function PushProcessor(client) {
            return false;
        }

-        const memberCount = Object.keys(room.currentState.members).filter(function(m) {
-            return room.currentState.members[m].membership == 'join';
-        }).length;
+        const memberCount = room.currentState.getJoinedMemberCount();

        const m = cond.is.match(/^([=<>]*)([0-9]*)$/);
        if (!m) {
@@ -194,30 +209,50 @@ function PushProcessor(client) {
    };

    const eventFulfillsEventMatchCondition = function(cond, ev) {
+        if (!cond.key) {
+            return false;
+        }
+
        const val = valueForDottedKey(cond.key, ev);
        if (!val || typeof val != 'string') {
            return false;
        }

-        let pat;
-        if (cond.key == 'content.body') {
-            pat = '(^|\\W)' + globToRegexp(cond.pattern) + '(\\W|$)';
-        } else {
-            pat = '^' + globToRegexp(cond.pattern) + '$';
+        if (cond.value) {
+            return cond.value === val;
        }
-        const regex = new RegExp(pat, 'i');
+
+        let regex;
+
+        if (cond.key == 'content.body') {
+            regex = createCachedRegex('(^|\\W)', cond.pattern, '(\\W|$)');
+        } else {
+            regex = createCachedRegex('^', cond.pattern, '$');
+        }
+
        return !!val.match(regex);
    };

+    const createCachedRegex = function(prefix, glob, suffix) {
+        if (cachedGlobToRegex[glob]) {
+            return cachedGlobToRegex[glob];
+        }
+        cachedGlobToRegex[glob] = new RegExp(
+            prefix + globToRegexp(glob) + suffix,
+            'i', // Case insensitive
+        );
+        return cachedGlobToRegex[glob];
+    };
+
    const globToRegexp = function(glob) {
        // From
        // https://github.com/matrix-org/synapse/blob/abbee6b29be80a77e05730707602f3bbfc3f38cb/synapse/push/__init__.py#L132
        // Because micromatch is about 130KB with dependencies,
        // and minimatch is not much better.
        let pat = escapeRegExp(glob);
-        pat = pat.replace(/\\\*/, '.*');
-        pat = pat.replace(/\?/, '.');
-        pat = pat.replace(/\\\[(!|)(.*)\\]/, function(match, p1, p2, offset, string) {
+        pat = pat.replace(/\\\*/g, '.*');
+        pat = pat.replace(/\?/g, '.');
+        pat = pat.replace(/\\\[(!|)(.*)\\]/g, function(match, p1, p2, offset, string) {
            const first = p1 && '^' || '';
            const second = p2.replace(/\\\-/, '-');
            return '[' + first + second + ']';
@@ -291,6 +326,17 @@ function PushProcessor(client) {
        return actionObj;
    };

+    this.ruleMatchesEvent = function(rule, ev) {
+        let ret = true;
+        for (let i = 0; i < rule.conditions.length; ++i) {
+            const cond = rule.conditions[i];
+            ret &= eventFulfillsCondition(cond, ev);
+        }
+        //console.log("Rule "+rule.rule_id+(ret ? " matches" : " doesn't match"));
+        return ret;
+    };
+
+
    /**
     * Get the user's push actions for the given event
     *
@@ -301,6 +347,27 @@ function PushProcessor(client) {
    this.actionsForEvent = function(ev) {
        return pushActionsForEventAndRulesets(ev, client.pushRules);
    };
+
+    /**
+     * Get one of the users push rules by its ID
+     *
+     * @param {string} ruleId The ID of the rule to search for
+     * @return {object} The push rule, or null if no such rule was found
+     */
+    this.getPushRuleById = function(ruleId) {
+        for (const scope of ['device', 'global']) {
+            if (client.pushRules[scope] === undefined) continue;
+
+            for (const kind of RULEKINDS_IN_ORDER) {
+                if (client.pushRules[scope][kind] === undefined) continue;
+
+                for (const rule of client.pushRules[scope][kind]) {
+                    if (rule.rule_id === ruleId) return rule;
+                }
+            }
+        }
+        return null;
+    };
 }

 /**
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,7 +19,7 @@ import Promise from 'bluebird';
 import SyncAccumulator from "../sync-accumulator";
 import utils from "../utils";

-const VERSION = 1;
+const VERSION = 3;

 function createDatabase(db) {
    // Make user store, clobber based on user ID. (userId property of User objects)
@@ -32,6 +33,20 @@ function createDatabase(db) {
    db.createObjectStore("sync", { keyPath: ["clobber"] });
 }

+function upgradeSchemaV2(db) {
+    const oobMembersStore = db.createObjectStore(
+        "oob_membership_events", {
+            keyPath: ["room_id", "state_key"],
+        });
+    oobMembersStore.createIndex("room", "room_id");
+}
+
+function upgradeSchemaV3(db) {
+    db.createObjectStore("client_options",
+        { keyPath: ["clobber"]});
+}
+
+
 /**
 * Helper method to collect results from a Cursor and promiseify it.
 * @param {ObjectStore|Index} store The store to perform openCursor on.
@@ -62,28 +77,39 @@ function selectQuery(store, keyRange, resultMapper) {
    });
 }

-function promiseifyTxn(txn) {
+function txnAsPromise(txn) {
    return new Promise((resolve, reject) => {
        txn.oncomplete = function(event) {
            resolve(event);
        };
        txn.onerror = function(event) {
-            reject(event);
+            reject(event.target.error);
        };
    });
 }

-function promiseifyRequest(req) {
+function reqAsEventPromise(req) {
    return new Promise((resolve, reject) => {
        req.onsuccess = function(event) {
            resolve(event);
        };
        req.onerror = function(event) {
-            reject(event);
+            reject(event.target.error);
        };
    });
 }

+function reqAsPromise(req) {
+    return new Promise((resolve, reject) => {
+        req.onsuccess = () => resolve(req);
+        req.onerror = (err) => reject(err);
+    });
+}
+
+function reqAsCursorPromise(req) {
+    return reqAsEventPromise(req).then((event) => event.target.result);
+}
+
 /**
 * Does the actual reading from and writing to the indexeddb
 *
@@ -101,7 +127,9 @@ const LocalIndexedDBStoreBackend = function LocalIndexedDBStoreBackend(
    this.indexedDB = indexedDBInterface;
    this._dbName = "matrix-js-sdk:" + (dbName || "default");
    this.db = null;
+    this._disconnected = true;
    this._syncAccumulator = new SyncAccumulator();
+    this._isNewlyCreated = false;
 };


@@ -112,15 +140,17 @@ LocalIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if successfully connected.
     */
    connect: function() {
-        if (this.db) {
+        if (!this._disconnected) {
            console.log(
-                `LocalIndexedDBStoreBackend.connect: already connected`,
+                `LocalIndexedDBStoreBackend.connect: already connected or connecting`,
            );
            return Promise.resolve();
        }

+        this._disconnected = false;
+
        console.log(
-            `LocalIndexedDBStoreBackend.connect: connecting`,
+            `LocalIndexedDBStoreBackend.connect: connecting...`,
        );
        const req = this.indexedDB.open(this._dbName, VERSION);
        req.onupgradeneeded = (ev) => {
@@ -130,8 +160,15 @@ LocalIndexedDBStoreBackend.prototype = {
                `LocalIndexedDBStoreBackend.connect: upgrading from ${oldVersion}`,
            );
            if (oldVersion < 1) { // The database did not previously exist.
+                this._isNewlyCreated = true;
                createDatabase(db);
            }
+            if (oldVersion < 2) {
+                upgradeSchemaV2(db);
+            }
+            if (oldVersion < 3) {
+                upgradeSchemaV3(db);
+            }
            // Expand as needed.
        };

@@ -142,9 +179,9 @@ LocalIndexedDBStoreBackend.prototype = {
        };

        console.log(
-            `LocalIndexedDBStoreBackend.connect: awaiting connection`,
+            `LocalIndexedDBStoreBackend.connect: awaiting connection...`,
        );
-        return promiseifyRequest(req).then((ev) => {
+        return reqAsEventPromise(req).then((ev) => {
            console.log(
                `LocalIndexedDBStoreBackend.connect: connected`,
            );
@@ -159,6 +196,10 @@ LocalIndexedDBStoreBackend.prototype = {
            return this._init();
        });
    },
+    /** @return {bool} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(this._isNewlyCreated);
+    },

    /**
     * Having connected, load initial data from the database and prepare for use
@@ -183,6 +224,124 @@ LocalIndexedDBStoreBackend.prototype = {
        });
    },

+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {Promise<event[]>} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return new Promise((resolve, reject) =>{
+            const tx = this.db.transaction(["oob_membership_events"], "readonly");
+            const store = tx.objectStore("oob_membership_events");
+            const roomIndex = store.index("room");
+            const range = IDBKeyRange.only(roomId);
+            const request = roomIndex.openCursor(range);
+
+            const membershipEvents = [];
+            // did we encounter the oob_written marker object
+            // amongst the results? That means OOB member
+            // loading already happened for this room
+            // but there were no members to persist as they
+            // were all known already
+            let oobWritten = false;
+
+            request.onsuccess = (event) => {
+                const cursor = event.target.result;
+                if (!cursor) {
+                    // Unknown room
+                    if (!membershipEvents.length && !oobWritten) {
+                        return resolve(null);
+                    }
+                    return resolve(membershipEvents);
+                }
+                const record = cursor.value;
+                if (record.oob_written) {
+                    oobWritten = true;
+                } else {
+                    membershipEvents.push(record);
+                }
+                cursor.continue();
+            };
+            request.onerror = (err) => {
+                reject(err);
+            };
+        }).then((events) => {
+            console.log(`LL: got ${events && events.length}` +
+                ` membershipEvents from storage for room ${roomId} ...`);
+            return events;
+        });
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     */
+    setOutOfBandMembers: async function(roomId, membershipEvents) {
+        console.log(`LL: backend about to store ${membershipEvents.length}` +
+            ` members for ${roomId}`);
+        const tx = this.db.transaction(["oob_membership_events"], "readwrite");
+        const store = tx.objectStore("oob_membership_events");
+        membershipEvents.forEach((e) => {
+            store.put(e);
+        });
+        // aside from all the events, we also write a marker object to the store
+        // to mark the fact that OOB members have been written for this room.
+        // It's possible that 0 members need to be written as all where previously know
+        // but we still need to know whether to return null or [] from getOutOfBandMembers
+        // where null means out of band members haven't been stored yet for this room
+        const markerObject = {
+            room_id: roomId,
+            oob_written: true,
+            state_key: 0,
+        };
+        store.put(markerObject);
+        await txnAsPromise(tx);
+        console.log(`LL: backend done storing for ${roomId}!`);
+    },
+
+    clearOutOfBandMembers: async function(roomId) {
+        // the approach to delete all members for a room
+        // is to get the min and max state key from the index
+        // for that room, and then delete between those
+        // keys in the store.
+        // this should be way faster than deleting every member
+        // individually for a large room.
+        const readTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readonly");
+        const store = readTx.objectStore("oob_membership_events");
+        const roomIndex = store.index("room");
+        const roomRange = IDBKeyRange.only(roomId);
+
+        const minStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "next"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const maxStateKeyProm = reqAsCursorPromise(
+                roomIndex.openKeyCursor(roomRange, "prev"),
+            ).then((cursor) => cursor && cursor.primaryKey[1]);
+        const [minStateKey, maxStateKey] = await Promise.all(
+            [minStateKeyProm, maxStateKeyProm]);
+
+        const writeTx = this.db.transaction(
+            ["oob_membership_events"],
+            "readwrite");
+        const writeStore = writeTx.objectStore("oob_membership_events");
+        const membersKeyRange = IDBKeyRange.bound(
+            [roomId, minStateKey],
+            [roomId, maxStateKey],
+        );
+
+        console.log(`LL: Deleting all users + marker in storage for ` +
+            `room ${roomId}, with key range:`,
+            [roomId, minStateKey], [roomId, maxStateKey]);
+        await reqAsPromise(writeStore.delete(membersKeyRange));
+    },
+
    /**
     * Clear the entire database. This should be used when logging out of a client
     * to prevent mixing data between accounts.
@@ -219,7 +378,7 @@ LocalIndexedDBStoreBackend.prototype = {

    /**
     * @param {boolean=} copy If false, the data returned is from internal
-     * buffers and must not be muated. Otherwise, a copy is made before
+     * buffers and must not be mutated. Otherwise, a copy is made before
     * returning such that the data can be safely mutated. Default: true.
     *
     * @return {Promise} Resolves with a sync response to restore the
@@ -240,6 +399,10 @@ LocalIndexedDBStoreBackend.prototype = {
        }
    },

+    getNextBatchToken: function() {
+        return Promise.resolve(this._syncAccumulator.getNextBatchToken());
+    },
+
    setSyncData: function(syncData) {
        return Promise.resolve().then(() => {
            this._syncAccumulator.accumulate(syncData);
@@ -276,7 +439,7 @@ LocalIndexedDBStoreBackend.prototype = {
                roomsData: roomsData,
                groupsData: groupsData,
            }); // put == UPSERT
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -293,7 +456,7 @@ LocalIndexedDBStoreBackend.prototype = {
            for (let i = 0; i < accountData.length; i++) {
                store.put(accountData[i]); // put == UPSERT
            }
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -315,7 +478,7 @@ LocalIndexedDBStoreBackend.prototype = {
                    event: tuple[1],
                }); // put == UPSERT
            }
-            return promiseifyTxn(txn);
+            return txnAsPromise(txn);
        });
    },

@@ -341,16 +504,18 @@ LocalIndexedDBStoreBackend.prototype = {
     */
    _loadAccountData: function() {
        console.log(
-            `LocalIndexedDBStoreBackend: loading account data`,
+            `LocalIndexedDBStoreBackend: loading account data...`,
        );
        return Promise.try(() => {
-            console.log(
-                `LocalIndexedDBStoreBackend: loaded account data`,
-            );
            const txn = this.db.transaction(["accountData"], "readonly");
            const store = txn.objectStore("accountData");
            return selectQuery(store, undefined, (cursor) => {
                return cursor.value;
+            }).then((result) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded account data`,
+                );
+                return result;
            });
        });
    },
@@ -361,17 +526,17 @@ LocalIndexedDBStoreBackend.prototype = {
     */
    _loadSyncData: function() {
        console.log(
-            `LocalIndexedDBStoreBackend: loaded sync data`,
+            `LocalIndexedDBStoreBackend: loading sync data...`,
        );
        return Promise.try(() => {
-            console.log(
-                `LocalIndexedDBStoreBackend: loaded sync data`,
-            );
            const txn = this.db.transaction(["sync"], "readonly");
            const store = txn.objectStore("sync");
            return selectQuery(store, undefined, (cursor) => {
                return cursor.value;
            }).then((results) => {
+                console.log(
+                    `LocalIndexedDBStoreBackend: loaded sync data`,
+                );
                if (results.length > 1) {
                    console.warn("loadSyncData: More than 1 sync row found.");
                }
@@ -379,6 +544,28 @@ LocalIndexedDBStoreBackend.prototype = {
            });
        });
    },
+
+    getClientOptions: function() {
+        return Promise.resolve().then(() => {
+            const txn = this.db.transaction(["client_options"], "readonly");
+            const store = txn.objectStore("client_options");
+            return selectQuery(store, undefined, (cursor) => {
+                if (cursor.value && cursor.value && cursor.value.options) {
+                    return cursor.value.options;
+                }
+            }).then((results) => results[0]);
+        });
+    },
+
+    storeClientOptions: async function(options) {
+        const txn = this.db.transaction(["client_options"], "readwrite");
+        const store = txn.objectStore("client_options");
+        store.put({
+            clobber: "-", // constant key so will always clobber
+            options: options,
+        }); // put == UPSERT
+        await txnAsPromise(txn);
+    },
 };

 export default LocalIndexedDBStoreBackend;
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -26,25 +27,23 @@ import Promise from 'bluebird';
 * @param {string} workerScript URL to the worker script
 * @param {string=} dbName Optional database name. The same name must be used
 * to open the same database.
- * @param {Object} WorkerApi The web worker compatible interface object
+ * @param {Object} workerApi The web worker compatible interface object
 */
 const RemoteIndexedDBStoreBackend = function RemoteIndexedDBStoreBackend(
-    workerScript, dbName, WorkerApi,
+    workerScript, dbName, workerApi,
 ) {
+    this._workerScript = workerScript;
    this._dbName = dbName;
-    this._worker = new WorkerApi(workerScript);
+    this._workerApi = workerApi;
+    this._worker = null;
    this._nextSeq = 0;
    // The currently in-flight requests to the actual backend
    this._inFlight = {
        // seq: promise,
    };
-
-    this._worker.onmessage = this._onWorkerMessage.bind(this);
-
-    // tell the worker the db name.
-    this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
-        console.log("IndexedDB worker is ready");
-    });
+    // Once we start connecting, we keep the promise and re-use it
+    // if we try to connect again
+    this._startPromise = null;
 };


@@ -55,7 +54,7 @@ RemoteIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolves if successfully connected.
     */
    connect: function() {
-        return this._startPromise.then(() => this._doCmd('connect'));
+        return this._ensureStarted().then(() => this._doCmd('connect'));
    },

    /**
@@ -64,9 +63,12 @@ RemoteIndexedDBStoreBackend.prototype = {
     * @return {Promise} Resolved when the database is cleared.
     */
    clearDatabase: function() {
-        return this._startPromise.then(() => this._doCmd('clearDatabase'));
+        return this._ensureStarted().then(() => this._doCmd('clearDatabase'));
+    },
+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return this._doCmd('isNewlyCreated');
    },
-
    /**
     * @return {Promise} Resolves with a sync response to restore the
     * client state to where it was at the last save, or null if there
@@ -76,6 +78,10 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('getSavedSync');
    },

+    getNextBatchToken: function() {
+        return this._doCmd('getNextBatchToken');
+    },
+
    setSyncData: function(syncData) {
        return this._doCmd('setSyncData', [syncData]);
    },
@@ -84,6 +90,40 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('syncToDatabase', [users]);
    },

+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return this._doCmd('getOutOfBandMembers', [roomId]);
+    },
+
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        return this._doCmd('setOutOfBandMembers', [roomId, membershipEvents]);
+    },
+
+    clearOutOfBandMembers: function(roomId) {
+        return this._doCmd('clearOutOfBandMembers', [roomId]);
+    },
+
+    getClientOptions: function() {
+        return this._doCmd('getClientOptions');
+    },
+
+    storeClientOptions: function(options) {
+        return this._doCmd('storeClientOptions', [options]);
+    },

    /**
     * Load all user presence events from the database. This is not cached.
@@ -93,6 +133,19 @@ RemoteIndexedDBStoreBackend.prototype = {
        return this._doCmd('getUserPresenceEvents');
    },

+    _ensureStarted: function() {
+        if (this._startPromise === null) {
+            this._worker = new this._workerApi(this._workerScript);
+            this._worker.onmessage = this._onWorkerMessage.bind(this);
+
+            // tell the worker the db name.
+            this._startPromise = this._doCmd('_setupWorker', [this._dbName]).then(() => {
+                console.log("IndexedDB worker is ready");
+            });
+        }
+        return this._startPromise;
+    },
+
    _doCmd: function(cmd, args) {
        // wrap in a q so if the postMessage throws,
        // the promise automatically gets rejected
@@ -131,7 +184,9 @@ RemoteIndexedDBStoreBackend.prototype = {
            if (msg.command == 'cmd_success') {
                def.resolve(msg.result);
            } else {
-                def.reject(msg.error);
+                const error = new Error(msg.error.message);
+                error.name = msg.error.name;
+                def.reject(error);
            }
        } else {
            console.warn("Unrecognised message from worker: " + msg);
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -66,6 +67,9 @@ class IndexedDBStoreWorker {
            case 'connect':
                prom = this.backend.connect();
                break;
+            case 'isNewlyCreated':
+                prom = this.backend.isNewlyCreated();
+                break;
            case 'clearDatabase':
                prom = this.backend.clearDatabase().then((result) => {
                    // This returns special classes which can't be cloned
@@ -88,10 +92,28 @@ class IndexedDBStoreWorker {
            case 'getUserPresenceEvents':
                prom = this.backend.getUserPresenceEvents();
                break;
+            case 'getNextBatchToken':
+                prom = this.backend.getNextBatchToken();
+                break;
+            case 'getOutOfBandMembers':
+                prom = this.backend.getOutOfBandMembers(msg.args[0]);
+                break;
+            case 'clearOutOfBandMembers':
+                prom = this.backend.clearOutOfBandMembers(msg.args[0]);
+                break;
+            case 'setOutOfBandMembers':
+                prom = this.backend.setOutOfBandMembers(msg.args[0], msg.args[1]);
+                break;
+            case 'getClientOptions':
+                prom = this.backend.getClientOptions();
+                break;
+            case 'storeClientOptions':
+                prom = this.backend.storeClientOptions(msg.args[0]);
+                break;
        }

        if (prom === undefined) {
-            postMessage({
+            this.postMessage({
                command: 'cmd_fail',
                seq: msg.seq,
                // Can't be an Error because they're not structured cloneable
@@ -113,7 +135,10 @@ class IndexedDBStoreWorker {
                command: 'cmd_fail',
                seq: msg.seq,
                // Just send a string because Error objects aren't cloneable
-                error: "Error running command",
+                error: {
+                    message: err.message,
+                    name: err.name,
+                },
            });
        });
    }
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -145,6 +146,19 @@ IndexedDBStore.prototype.getSavedSync = function() {
    return this.backend.getSavedSync();
 };

+/** @return {Promise<bool>} whether or not the database was newly created in this session. */
+IndexedDBStore.prototype.isNewlyCreated = function() {
+    return this.backend.isNewlyCreated();
+};
+
+/**
+ * @return {Promise} If there is a saved sync, the nextBatch token
+ * for this sync, otherwise null.
+ */
+IndexedDBStore.prototype.getSavedSyncToken = function() {
+    return this.backend.getNextBatchToken();
+},
+
 /**
 * Delete all data from this store.
 * @return {Promise} Resolves if the data was deleted from the database.
@@ -159,13 +173,27 @@ IndexedDBStore.prototype.deleteAllData = function() {
    });
 };

+/**
+ * Whether this store would like to save its data
+ * Note that obviously whether the store wants to save or
+ * not could change between calling this function and calling
+ * save().
+ *
+ * @return {boolean} True if calling save() will actually save
+ *     (at the time this function is called).
+ */
+IndexedDBStore.prototype.wantsSave = function() {
+    const now = Date.now();
+    return now - this._syncTs > WRITE_DELAY_MS;
+};
+
 /**
 * Possibly write data to the database.
- * @return {Promise} Promise resolves after the write completes.
+ * @return {Promise} Promise resolves after the write completes
+ *     (or immediately if no write is performed)
 */
 IndexedDBStore.prototype.save = function() {
-    const now = Date.now();
-    if (now - this._syncTs > WRITE_DELAY_MS) {
+    if (this.wantsSave()) {
        return this._reallySave();
    }
    return Promise.resolve();
@@ -196,4 +224,39 @@ IndexedDBStore.prototype.setSyncData = function(syncData) {
    return this.backend.setSyncData(syncData);
 };

+/**
+ * Returns the out-of-band membership events for this room that
+ * were previously loaded.
+ * @param {string} roomId
+ * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+ * @returns {null} in case the members for this room haven't been stored yet
+ */
+IndexedDBStore.prototype.getOutOfBandMembers = function(roomId) {
+    return this.backend.getOutOfBandMembers(roomId);
+};
+
+/**
+ * Stores the out-of-band membership events for this room. Note that
+ * it still makes sense to store an empty array as the OOB status for the room is
+ * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+ * @param {string} roomId
+ * @param {event[]} membershipEvents the membership events to store
+ * @returns {Promise} when all members have been stored
+ */
+IndexedDBStore.prototype.setOutOfBandMembers = function(roomId, membershipEvents) {
+    return this.backend.setOutOfBandMembers(roomId, membershipEvents);
+};
+
+IndexedDBStore.prototype.clearOutOfBandMembers = function(roomId) {
+    return this.backend.clearOutOfBandMembers(roomId);
+};
+
+IndexedDBStore.prototype.getClientOptions = function() {
+    return this.backend.getClientOptions();
+};
+
+IndexedDBStore.prototype.storeClientOptions = function(options) {
+    return this.backend.storeClientOptions(options);
+};
+
 module.exports.IndexedDBStore = IndexedDBStore;
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -51,6 +52,10 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) {
        // type : content
    };
    this.localStorage = opts.localStorage;
+    this._oobMembers = {
+        // roomId: [member events]
+    };
+    this._clientOptions = {};
 };

 module.exports.MatrixInMemoryStore.prototype = {
@@ -63,6 +68,10 @@ module.exports.MatrixInMemoryStore.prototype = {
        return this.syncToken;
    },

+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },

    /**
     * Set the token to stream from.
@@ -315,6 +324,15 @@ module.exports.MatrixInMemoryStore.prototype = {
        return Promise.resolve();
    },

+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
    /**
     * Save does nothing as there is no backing data store.
     */
@@ -337,6 +355,14 @@ module.exports.MatrixInMemoryStore.prototype = {
        return Promise.resolve(null);
    },

+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
    /**
     * Delete all data from this store.
     * @return {Promise} An immediately resolved promise.
@@ -359,4 +385,35 @@ module.exports.MatrixInMemoryStore.prototype = {
        };
        return Promise.resolve();
    },
+    /**
+     * Returns the out-of-band membership events for this room that
+     * were previously loaded.
+     * @param {string} roomId
+     * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members
+     * @returns {null} in case the members for this room haven't been stored yet
+     */
+    getOutOfBandMembers: function(roomId) {
+        return Promise.resolve(this._oobMembers[roomId] || null);
+    },
+    /**
+     * Stores the out-of-band membership events for this room. Note that
+     * it still makes sense to store an empty array as the OOB status for the room is
+     * marked as fetched, and getOutOfBandMembers will return an empty array instead of null
+     * @param {string} roomId
+     * @param {event[]} membershipEvents the membership events to store
+     * @returns {Promise} when all members have been stored
+     */
+    setOutOfBandMembers: function(roomId, membershipEvents) {
+        this._oobMembers[roomId] = membershipEvents;
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve(this._clientOptions);
+    },
+
+    storeClientOptions: function(options) {
+        this._clientOptions = Object.assign({}, options);
+        return Promise.resolve();
+    },
 };
@@ -1,5 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -48,17 +50,18 @@ function WebStorageSessionStore(webStore) {
 }

 WebStorageSessionStore.prototype = {
-
    /**
-     * Store the end to end account for the logged-in user.
-     * @param {string} account Base64 encoded account.
+     * Remove the stored end to end account for the logged-in user.
     */
-    storeEndToEndAccount: function(account) {
-        this.store.setItem(KEY_END_TO_END_ACCOUNT, account);
+    removeEndToEndAccount: function() {
+        this.store.removeItem(KEY_END_TO_END_ACCOUNT);
    },

    /**
     * Load the end to end account for the logged-in user.
+     * Note that the end-to-end account is now stored in the
+     * crypto store rather than here: this remains here so
+     * old sessions can be migrated out of the session store.
     * @return {?string} Base64 encoded account.
     */
    getEndToEndAccount: function() {
@@ -66,43 +69,24 @@ WebStorageSessionStore.prototype = {
    },

    /**
-     * Stores the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @param {object} devices A map from device ID to keys for the device.
+     * Retrieves the known devices for all users.
+     * @return {object} A map from user ID to map of device ID to keys for the device.
     */
-    storeEndToEndDevicesForUser: function(userId, devices) {
-        setJsonItem(this.store, keyEndToEndDevicesForUser(userId), devices);
-    },
-
-    /**
-     * Retrieves the known devices for a user.
-     * @param {string} userId The user's ID.
-     * @return {object} A map from device ID to keys for the device.
-     */
-    getEndToEndDevicesForUser: function(userId) {
-        return getJsonItem(this.store, keyEndToEndDevicesForUser(userId));
-    },
-
-    storeEndToEndDeviceTrackingStatus: function(statusMap) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS, statusMap);
+    getAllEndToEndDevices: function() {
+        const prefix = keyEndToEndDevicesForUser('');
+        const devices = {};
+        for (let i = 0; i < this.store.length; ++i) {
+            const key = this.store.key(i);
+            const userId = key.substr(prefix.length);
+            if (key.startsWith(prefix)) devices[userId] = getJsonItem(this.store, key);
+        }
+        return devices;
    },

    getEndToEndDeviceTrackingStatus: function() {
        return getJsonItem(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
    },

-    /**
-     * Store the sync token corresponding to the device list.
-     *
-     * This is used when starting the client, to get a list of the users who
-     * have changed their device list since the list time we were running.
-     *
-     * @param {String?} token
-     */
-    storeEndToEndDeviceSyncToken: function(token) {
-        setJsonItem(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN, token);
-    },
-
    /**
     * Get the sync token corresponding to the device list.
     *
@@ -113,17 +97,12 @@ WebStorageSessionStore.prototype = {
    },

    /**
-     * Store a session between the logged-in user and another device
-     * @param {string} deviceKey The public key of the other device.
-     * @param {string} sessionId The ID for this end-to-end session.
-     * @param {string} session Base64 encoded end-to-end session.
+     * Removes all end to end device data from the store
     */
-    storeEndToEndSession: function(deviceKey, sessionId, session) {
-        const sessions = this.getEndToEndSessions(deviceKey) || {};
-        sessions[sessionId] = session;
-        setJsonItem(
-            this.store, keyEndToEndSessions(deviceKey), sessions,
-        );
+    removeEndToEndDeviceData: function() {
+        removeByPrefix(this.store, keyEndToEndDevicesForUser(''));
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_LIST_TRACKING_STATUS);
+        removeByPrefix(this.store, KEY_END_TO_END_DEVICE_SYNC_TOKEN);
    },

    /**
@@ -136,6 +115,29 @@ WebStorageSessionStore.prototype = {
        return getJsonItem(this.store, keyEndToEndSessions(deviceKey));
    },

+    /**
+     * Retrieve all end-to-end sessions between the logged-in user and other
+     * devices.
+     * @return {object} A map of {deviceKey -> {sessionId -> session pickle}}
+     */
+    getAllEndToEndSessions: function() {
+        const deviceKeys = getKeysWithPrefix(this.store, keyEndToEndSessions(''));
+        const results = {};
+        for (const k of deviceKeys) {
+            const unprefixedKey = k.substr(keyEndToEndSessions('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
+    },
+
+    /**
+     * Remove all end-to-end sessions from the store
+     * This is used after migrating sessions awat from the sessions store.
+     */
+    removeAllEndToEndSessions: function() {
+        removeByPrefix(this.store, keyEndToEndSessions(''));
+    },
+
    /**
     * Retrieve a list of all known inbound group sessions
     *
@@ -167,27 +169,26 @@ WebStorageSessionStore.prototype = {
        return this.store.getItem(key);
    },

-    storeEndToEndInboundGroupSession: function(senderKey, sessionId, pickledSession) {
-        const key = keyEndToEndInboundGroupSession(senderKey, sessionId);
-        return this.store.setItem(key, pickledSession);
+    removeAllEndToEndInboundGroupSessions: function() {
+        removeByPrefix(this.store, E2E_PREFIX + 'inboundgroupsessions/');
    },

    /**
-     * Store the end-to-end state for a room.
-     * @param {string} roomId The room's ID.
-     * @param {object} roomInfo The end-to-end info for the room.
+     * Get the end-to-end state for all rooms
+     * @return {object} roomId -> object with the end-to-end info for the room.
     */
-    storeEndToEndRoom: function(roomId, roomInfo) {
-        setJsonItem(this.store, keyEndToEndRoom(roomId), roomInfo);
+    getAllEndToEndRooms: function() {
+        const roomKeys = getKeysWithPrefix(this.store, keyEndToEndRoom(''));
+        const results = {};
+        for (const k of roomKeys) {
+            const unprefixedKey = k.substr(keyEndToEndRoom('').length);
+            results[unprefixedKey] = getJsonItem(this.store, k);
+        }
+        return results;
    },

-    /**
-     * Get the end-to-end state for a room
-     * @param {string} roomId The room's ID.
-     * @return {object} The end-to-end info for the room.
-     */
-    getEndToEndRoom: function(roomId) {
-        return getJsonItem(this.store, keyEndToEndRoom(roomId));
+    removeAllEndToEndRooms: function() {
+        removeByPrefix(this.store, keyEndToEndRoom(''));
    },
 };

@@ -223,8 +224,24 @@ function getJsonItem(store, key) {
    return null;
 }

-function setJsonItem(store, key, val) {
-    store.setItem(key, JSON.stringify(val));
+function getKeysWithPrefix(store, prefix) {
+    const results = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) results.push(key);
+    }
+    return results;
+}
+
+function removeByPrefix(store, prefix) {
+    const toRemove = [];
+    for (let i = 0; i < store.length; ++i) {
+        const key = store.key(i);
+        if (key.startsWith(prefix)) toRemove.push(key);
+    }
+    for (const key of toRemove) {
+        store.removeItem(key);
+    }
 }

 function debuglog() {
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -31,6 +32,11 @@ function StubStore() {

 StubStore.prototype = {

+    /** @return {Promise<bool>} whether or not the database was newly created in this session. */
+    isNewlyCreated: function() {
+        return Promise.resolve(true);
+    },
+
    /**
     * Get the sync token.
     * @return {string}
@@ -216,6 +222,15 @@ StubStore.prototype = {
        return Promise.resolve();
    },

+    /**
+     * We never want to save becase we have nothing to save to.
+     *
+     * @return {boolean} If the store wants to save
+     */
+    wantsSave: function() {
+        return false;
+    },
+
    /**
     * Save does nothing as there is no backing data store.
     */
@@ -238,6 +253,14 @@ StubStore.prototype = {
        return Promise.resolve(null);
    },

+    /**
+     * @return {Promise} If there is a saved sync, the nextBatch token
+     * for this sync, otherwise null.
+     */
+    getSavedSyncToken: function() {
+        return Promise.resolve(null);
+    },
+
    /**
     * Delete all data from this store. Does nothing since this store
     * doesn't store anything.
@@ -246,6 +269,26 @@ StubStore.prototype = {
    deleteAllData: function() {
        return Promise.resolve();
    },
+
+    getOutOfBandMembers: function() {
+        return Promise.resolve(null);
+    },
+
+    setOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    clearOutOfBandMembers: function() {
+        return Promise.resolve();
+    },
+
+    getClientOptions: function() {
+        return Promise.resolve();
+    },
+
+    storeClientOptions: function() {
+        return Promise.resolve();
+    },
 };

 /** Stub Store class. */
@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -62,6 +63,11 @@ class SyncAccumulator {
            //       { event: $event, token: null|token },
            //       ...
            //    ],
+            //    _summary: {
+            //       m.heroes: [ $user_id ],
+            //       m.joined_member_count: $count,
+            //       m.invited_member_count: $count
+            //    },
            //    _accountData: { $event_type: json },
            //    _unreadNotifications: { ... unread_notifications JSON ... },
            //    _readReceipts: { $user_id: { data: $json, eventId: $event_id }}
@@ -241,6 +247,7 @@ class SyncAccumulator {
                _timeline: [],
                _accountData: Object.create(null),
                _unreadNotifications: {},
+                _summary: {},
                _readReceipts: {},
            };
        }
@@ -257,6 +264,17 @@ class SyncAccumulator {
        if (data.unread_notifications) {
            currentData._unreadNotifications = data.unread_notifications;
        }
+        if (data.summary) {
+            const HEROES_KEY = "m.heroes";
+            const INVITED_COUNT_KEY = "m.invited_member_count";
+            const JOINED_COUNT_KEY = "m.joined_member_count";
+
+            const acc = currentData._summary;
+            const sum = data.summary;
+            acc[HEROES_KEY] = sum[HEROES_KEY] || acc[HEROES_KEY];
+            acc[JOINED_COUNT_KEY] = sum[JOINED_COUNT_KEY] || acc[JOINED_COUNT_KEY];
+            acc[INVITED_COUNT_KEY] = sum[INVITED_COUNT_KEY] || acc[INVITED_COUNT_KEY];
+        }

        if (data.ephemeral && data.ephemeral.events) {
            data.ephemeral.events.forEach((e) => {
@@ -427,6 +445,7 @@ class SyncAccumulator {
                    prev_batch: null,
                },
                unread_notifications: roomData._unreadNotifications,
+                summary: roomData._summary,
            };
            // Add account data
            Object.keys(roomData._accountData).forEach((evType) => {
@@ -520,6 +539,10 @@ class SyncAccumulator {
            accountData: accData,
        };
    }
+
+    getNextBatchToken() {
+        return this.nextBatch;
+    }
 }

 function setState(eventMap, event) {
@@ -1,6 +1,7 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
+Copyright 2018 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -32,6 +33,8 @@ const utils = require("./utils");
 const Filter = require("./filter");
 const EventTimeline = require("./models/event-timeline");

+import {InvalidStoreError} from './errors';
+
 const DEBUG = true;

 // /sync requests allow you to set a timeout= but the request may continue
@@ -71,6 +74,8 @@ function debuglog(...params) {
 * SAFELY remove events from this room. It may not be safe to remove events if
 * there are other references to the timelines for this room.
 * Default: returns false.
+ * @param {Boolean=} opts.disablePresence True to perform syncing without automatically
+ * updating presence.
 */
 function SyncApi(client, opts) {
    this.client = client;
@@ -90,12 +95,14 @@ function SyncApi(client, opts) {
    this._peekRoomId = null;
    this._currentSyncRequest = null;
    this._syncState = null;
+    this._syncStateData = null; // additional data (eg. error object for failed sync)
    this._catchingUp = false;
    this._running = false;
    this._keepAliveTimer = null;
    this._connectionReturnedDefer = null;
    this._notifEvents = []; // accumulator of sync events in the current sync response
    this._failedSyncCount = 0; // Number of consecutive failed /sync requests
+    this._storeIsInvalid = false; // flag set if the store needs to be cleared before we can start

    if (client.getNotifTimelineSet()) {
        client.reEmitter.reEmit(client.getNotifTimelineSet(),
@@ -109,7 +116,8 @@ function SyncApi(client, opts) {
 */
 SyncApi.prototype.createRoom = function(roomId) {
    const client = this.client;
-    const room = new Room(roomId, {
+    const room = new Room(roomId, client, client.getUserId(), {
+        lazyLoadMembers: this.opts.lazyLoadMembers,
        pendingEventOrdering: this.opts.pendingEventOrdering,
        timelineSupport: client.timelineSupport,
    });
@@ -118,6 +126,7 @@ SyncApi.prototype.createRoom = function(roomId) {
                          "Room.timelineReset",
                          "Room.localEchoUpdated",
                          "Room.accountData",
+                          "Room.myMembership",
                         ]);
    this._registerStateListeners(room);
    return room;
@@ -131,6 +140,7 @@ SyncApi.prototype.createGroup = function(groupId) {
    const client = this.client;
    const group = new Group(groupId);
    client.reEmitter.reEmit(group, ["Group.profile", "Group.myMembership"]);
+    client.store.storeGroup(group);
    return group;
 };

@@ -227,9 +237,11 @@ SyncApi.prototype.syncLeftRooms = function() {

            self._processRoomEvents(room, stateEvents, timelineEvents);

-            room.recalculate(client.credentials.userId);
+            room.recalculate();
            client.store.storeRoom(room);
            client.emit("Room", room);
+
+            self._processEventsForNotifs(room, timelineEvents);
        });
        return rooms;
    });
@@ -296,7 +308,7 @@ SyncApi.prototype.peek = function(roomId) {
        peekRoom.currentState.setStateEvents(stateEvents);

        self._resolveInvites(peekRoom);
-        peekRoom.recalculate(self.client.credentials.userId);
+        peekRoom.recalculate();

        // roll backwards to diverge old state. addEventsToTimeline
        // will overwrite the pagination token, so make sure it overwrites
@@ -390,6 +402,49 @@ SyncApi.prototype.getSyncState = function() {
    return this._syncState;
 };

+/**
+ * Returns the additional data object associated with
+ * the current sync state, or null if there is no
+ * such data.
+ * Sync errors, if available, are put in the 'error' key of
+ * this object.
+ * @return {?Object}
+ */
+SyncApi.prototype.getSyncStateData = function() {
+    return this._syncStateData;
+};
+
+SyncApi.prototype.recoverFromSyncStartupError = async function(savedSyncPromise, err) {
+    // Wait for the saved sync to complete - we send the pushrules and filter requests
+    // before the saved sync has finished so they can run in parallel, but only process
+    // the results after the saved sync is done. Equivalently, we wait for it to finish
+    // before reporting failures from these functions.
+    await savedSyncPromise;
+    const keepaliveProm = this._startKeepAlives();
+    this._updateSyncState("ERROR", { error: err });
+    await keepaliveProm;
+};
+
+/**
+ * Is the lazy loading option different than in previous session?
+ * @param {bool} lazyLoadMembers current options for lazy loading
+ * @return {bool} whether or not the option has changed compared to the previous session */
+SyncApi.prototype._wasLazyLoadingToggled = async function(lazyLoadMembers) {
+    lazyLoadMembers = !!lazyLoadMembers;
+    // assume it was turned off before
+    // if we don't know any better
+    let lazyLoadMembersBefore = false;
+    const isStoreNewlyCreated = await this.client.store.isNewlyCreated();
+    if (!isStoreNewlyCreated) {
+        const prevClientOptions = await this.client.store.getClientOptions();
+        if (prevClientOptions) {
+            lazyLoadMembersBefore = !!prevClientOptions.lazyLoadMembers;
+        }
+        return lazyLoadMembersBefore !== lazyLoadMembers;
+    }
+    return false;
+};
+
 /**
 * Main entry point
 */
@@ -404,26 +459,72 @@ SyncApi.prototype.sync = function() {
        global.document.addEventListener("online", this._onOnlineBound, false);
    }

+    let savedSyncPromise = Promise.resolve();
+    let savedSyncToken = null;
+
    // We need to do one-off checks before we can begin the /sync loop.
    // These are:
    //   1) We need to get push rules so we can check if events should bing as we get
    //      them from /sync.
    //   2) We need to get/create a filter which we can use for /sync.
+    //   3) We need to check the lazy loading option matches what was used in the
+    //       stored sync. If it doesn't, we can't use the stored sync.

-    function getPushRules() {
-        client.getPushRules().done(function(result) {
+    async function getPushRules() {
+        try {
+            const result = await client.getPushRules();
            debuglog("Got push rules");
+
            client.pushRules = result;
-            getFilter(); // Now get the filter
-        }, function(err) {
-            self._startKeepAlives().done(function() {
-                getPushRules();
-            });
-            self._updateSyncState("ERROR", { error: err });
-        });
+        } catch (err) {
+            // wait for saved sync to complete before doing anything else,
+            // otherwise the sync state will end up being incorrect
+            await self.recoverFromSyncStartupError(savedSyncPromise, err);
+            getPushRules();
+            return;
+        }
+        checkLazyLoadStatus(); // advance to the next stage
    }

-    function getFilter() {
+    const checkLazyLoadStatus = async () => {
+        if (this.opts.lazyLoadMembers && client.isGuest()) {
+            this.opts.lazyLoadMembers = false;
+        }
+        if (this.opts.lazyLoadMembers) {
+            const supported = await client.doesServerSupportLazyLoading();
+            if (supported) {
+                this.opts.filter = await client.createFilter(
+                    Filter.LAZY_LOADING_SYNC_FILTER,
+                );
+            } else {
+                console.log("LL: lazy loading requested but not supported " +
+                    "by server, so disabling");
+                this.opts.lazyLoadMembers = false;
+            }
+        }
+        // need to vape the store when enabling LL and wasn't enabled before
+        const shouldClear = await this._wasLazyLoadingToggled(this.opts.lazyLoadMembers);
+        if (shouldClear) {
+            this._storeIsInvalid = true;
+            const reason = InvalidStoreError.TOGGLED_LAZY_LOADING;
+            const error = new InvalidStoreError(reason, !!this.opts.lazyLoadMembers);
+            this._updateSyncState("ERROR", { error });
+            // bail out of the sync loop now: the app needs to respond to this error.
+            // we leave the state as 'ERROR' which isn't great since this normally means
+            // we're retrying. The client must be stopped before clearing the stores anyway
+            // so the app should stop the client, clear the store and start it again.
+            console.warn("InvalidStoreError: store is not usable: stopping sync.");
+            return;
+        }
+        if (this.opts.lazyLoadMembers && this.opts.crypto) {
+            this.opts.crypto.enableLazyLoading();
+        }
+        await this.client._storeClientOptions();
+
+        getFilter(); // Now get the filter and start syncing
+    };
+
+    async function getFilter() {
        let filter;
        if (self.opts.filter) {
            filter = self.opts.filter;
@@ -432,28 +533,54 @@ SyncApi.prototype.sync = function() {
            filter.setTimelineLimit(self.opts.initialSyncLimit);
        }

-        client.getOrCreateFilter(
-            getFilterName(client.credentials.userId), filter,
-        ).done(function(filterId) {
-            // reset the notifications timeline to prepare it to paginate from
-            // the current point in time.
-            // The right solution would be to tie /sync pagination tokens into
-            // /notifications API somehow.
-            client.resetNotifTimelineSet();
+        let filterId;
+        try {
+            filterId = await client.getOrCreateFilter(
+                getFilterName(client.credentials.userId), filter,
+            );
+        } catch (err) {
+            // wait for saved sync to complete before doing anything else,
+            // otherwise the sync state will end up being incorrect
+            await self.recoverFromSyncStartupError(savedSyncPromise, err);
+            getFilter();
+            return;
+        }
+        // reset the notifications timeline to prepare it to paginate from
+        // the current point in time.
+        // The right solution would be to tie /sync pagination tokens into
+        // /notifications API somehow.
+        client.resetNotifTimelineSet();

-            self._sync({ filterId: filterId });
-        }, function(err) {
-            self._startKeepAlives().done(function() {
-                getFilter();
-            });
-            self._updateSyncState("ERROR", { error: err });
-        });
+        if (self._currentSyncRequest === null) {
+            // Send this first sync request here so we can then wait for the saved
+            // sync data to finish processing before we process the results of this one.
+            console.log("Sending first sync request...");
+            self._currentSyncRequest = self._doSyncRequest({ filterId }, savedSyncToken);
+        }
+
+        // Now wait for the saved sync to finish...
+        await savedSyncPromise;
+        self._sync({ filterId });
    }

    if (client.isGuest()) {
        // no push rules for guests, no access to POST filter for guests.
        self._sync({});
    } else {
+        // Pull the saved sync token out first, before the worker starts sending
+        // all the sync data which could take a while. This will let us send our
+        // first incremental sync request before we've processed our saved data.
+        savedSyncPromise = client.store.getSavedSyncToken().then((tok) => {
+            savedSyncToken = tok;
+            return client.store.getSavedSync();
+        }).then((savedSync) => {
+            if (savedSync) {
+                return self._syncFromCache(savedSync);
+            }
+        });
+        // Now start the first incremental sync request: this can also
+        // take a while so if we set it going now, we can wait for it
+        // to finish while we process our saved sync data.
        getPushRules();
    }
 };
@@ -489,6 +616,48 @@ SyncApi.prototype.retryImmediately = function() {
    this._startKeepAlives(0);
    return true;
 };
+/**
+ * Process a single set of cached sync data.
+ * @param {Object} savedSync a saved sync that was persisted by a store. This
+ * should have been acquired via client.store.getSavedSync().
+ */
+SyncApi.prototype._syncFromCache = async function(savedSync) {
+    debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
+
+    const nextSyncToken = savedSync.nextBatch;
+
+    // Set sync token for future incremental syncing
+    this.client.store.setSyncToken(nextSyncToken);
+
+    // No previous sync, set old token to null
+    const syncEventData = {
+        oldSyncToken: null,
+        nextSyncToken,
+        catchingUp: false,
+    };
+
+    const data = {
+        next_batch: nextSyncToken,
+        rooms: savedSync.roomsData,
+        groups: savedSync.groupsData,
+        account_data: {
+            events: savedSync.accountData,
+        },
+    };
+
+    try {
+        await this._processSyncResponse(syncEventData, data);
+    } catch(e) {
+        console.error("Error processing cached sync", e.stack || e);
+    }
+
+    // Don't emit a prepared if we've bailed because the store is invalid:
+    // in this case the client will not be usable until stopped & restarted
+    // so this would be useless and misleading.
+    if (!this._storeIsInvalid) {
+        this._updateSyncState("PREPARED", syncEventData);
+    }
+};

 /**
 * Invoke me to do /sync calls
@@ -509,13 +678,99 @@ SyncApi.prototype._sync = async function(syncOptions) {
        return;
    }

-    let filterId = syncOptions.filterId;
-    if (client.isGuest() && !filterId) {
-        filterId = this._getGuestFilter();
-    }
-
    const syncToken = client.store.getSyncToken();

+    let data;
+    try {
+        //debuglog('Starting sync since=' + syncToken);
+        if (this._currentSyncRequest === null) {
+            this._currentSyncRequest = this._doSyncRequest(syncOptions, syncToken);
+        }
+        data = await this._currentSyncRequest;
+    } catch (e) {
+        this._onSyncError(e, syncOptions);
+        return;
+    } finally {
+        this._currentSyncRequest = null;
+    }
+
+    //debuglog('Completed sync, next_batch=' + data.next_batch);
+
+    // set the sync token NOW *before* processing the events. We do this so
+    // if something barfs on an event we can skip it rather than constantly
+    // polling with the same token.
+    client.store.setSyncToken(data.next_batch);
+
+    // Reset after a successful sync
+    this._failedSyncCount = 0;
+
+    await client.store.setSyncData(data);
+
+    const syncEventData = {
+        oldSyncToken: syncToken,
+        nextSyncToken: data.next_batch,
+        catchingUp: this._catchingUp,
+    };
+
+    if (this.opts.crypto) {
+        // tell the crypto module we're about to process a sync
+        // response
+        await this.opts.crypto.onSyncWillProcess(syncEventData);
+    }
+
+    try {
+        await this._processSyncResponse(syncEventData, data);
+    } catch(e) {
+        // log the exception with stack if we have it, else fall back
+        // to the plain description
+        console.error("Caught /sync error", e.stack || e);
+    }
+
+    // update this as it may have changed
+    syncEventData.catchingUp = this._catchingUp;
+
+    // emit synced events
+    if (!syncOptions.hasSyncedBefore) {
+        this._updateSyncState("PREPARED", syncEventData);
+        syncOptions.hasSyncedBefore = true;
+    }
+
+    // tell the crypto module to do its processing. It may block (to do a
+    // /keys/changes request).
+    if (this.opts.crypto) {
+        await this.opts.crypto.onSyncCompleted(syncEventData);
+    }
+
+    // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
+    this._updateSyncState("SYNCING", syncEventData);
+
+    if (client.store.wantsSave()) {
+        // We always save the device list (if it's dirty) before saving the sync data:
+        // this means we know the saved device list data is at least as fresh as the
+        // stored sync data which means we don't have to worry that we may have missed
+        // device changes. We can also skip the delay since we're not calling this very
+        // frequently (and we don't really want to delay the sync for it).
+        if (this.opts.crypto) {
+            await this.opts.crypto.saveDeviceList(0);
+        }
+
+        // tell databases that everything is now in a consistent state and can be saved.
+        client.store.save();
+    }
+
+    // Begin next sync
+    this._sync(syncOptions);
+};
+
+SyncApi.prototype._doSyncRequest = function(syncOptions, syncToken) {
+    const qps = this._getSyncParams(syncOptions, syncToken);
+    return this.client._http.authedRequest(
+        undefined, "GET", "/sync", qps, undefined,
+        qps.timeout + BUFFER_PERIOD_MS,
+    );
+};
+
+SyncApi.prototype._getSyncParams = function(syncOptions, syncToken) {
    let pollTimeout = this.opts.pollTimeout;

    if (this.getSyncState() !== 'SYNCING' || this._catchingUp) {
@@ -534,14 +789,20 @@ SyncApi.prototype._sync = async function(syncOptions) {
        pollTimeout = 0;
    }

-    // normal timeout= plus buffer time
-    const clientSideTimeoutMs = pollTimeout + BUFFER_PERIOD_MS;
+    let filterId = syncOptions.filterId;
+    if (this.client.isGuest() && !filterId) {
+        filterId = this._getGuestFilter();
+    }

    const qps = {
        filter: filterId,
        timeout: pollTimeout,
    };

+    if (this.opts.disablePresence) {
+        qps.set_presence = "offline";
+    }
+
    if (syncToken) {
        qps.since = syncToken;
    } else {
@@ -559,97 +820,7 @@ SyncApi.prototype._sync = async function(syncOptions) {
        qps.timeout = 0;
    }

-    let savedSync;
-    if (!syncOptions.hasSyncedBefore) {
-        // Don't do an HTTP hit to /sync. Instead, load up the persisted /sync data,
-        // if there is data there.
-        savedSync = await client.store.getSavedSync();
-    }
-
-    let isCachedResponse = false;
-    let data;
-
-    if (savedSync) {
-        debuglog("sync(): not doing HTTP hit, instead returning stored /sync data");
-        isCachedResponse = true;
-        data = {
-            next_batch: savedSync.nextBatch,
-            rooms: savedSync.roomsData,
-            groups: savedSync.groupsData,
-            account_data: {
-                events: savedSync.accountData,
-            },
-        };
-    } else {
-        try {
-            //debuglog('Starting sync since=' + syncToken);
-            this._currentSyncRequest = client._http.authedRequest(
-                undefined, "GET", "/sync", qps, undefined, clientSideTimeoutMs,
-            );
-            data = await this._currentSyncRequest;
-        } catch (e) {
-            this._onSyncError(e, syncOptions);
-            return;
-        }
-    }
-
-    //debuglog('Completed sync, next_batch=' + data.next_batch);
-
-    // set the sync token NOW *before* processing the events. We do this so
-    // if something barfs on an event we can skip it rather than constantly
-    // polling with the same token.
-    client.store.setSyncToken(data.next_batch);
-
-    // Reset after a successful sync
-    this._failedSyncCount = 0;
-
-    // We need to wait until the sync data has been sent to the backend
-    // because it appears that the sync data gets modified somewhere in
-    // processing it in such a way as to make it no longer cloneable.
-    // XXX: Find out what is modifying it!
-    if (!isCachedResponse) {
-        // Don't give the store back its own cached data
-        await client.store.setSyncData(data);
-    }
-
-    try {
-        await this._processSyncResponse(syncToken, data);
-    } catch(e) {
-        // log the exception with stack if we have it, else fall back
-        // to the plain description
-        console.error("Caught /sync error", e.stack || e);
-    }
-
-    // emit synced events
-    const syncEventData = {
-        oldSyncToken: syncToken,
-        nextSyncToken: data.next_batch,
-        catchingUp: this._catchingUp,
-    };
-
-    if (!syncOptions.hasSyncedBefore) {
-        this._updateSyncState("PREPARED", syncEventData);
-        syncOptions.hasSyncedBefore = true;
-    }
-
-    if (!isCachedResponse) {
-        // tell the crypto module to do its processing. It may block (to do a
-        // /keys/changes request).
-        if (this.opts.crypto) {
-            await this.opts.crypto.onSyncCompleted(syncEventData);
-        }
-
-        // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates
-        this._updateSyncState("SYNCING", syncEventData);
-
-        // tell databases that everything is now in a consistent state and can be
-        // saved (no point doing so if we only have the data we just got out of the
-        // store).
-        client.store.save();
-    }
-
-    // Begin next sync
-    this._sync(syncOptions);
+    return qps;
 };

 SyncApi.prototype._onSyncError = function(err, syncOptions) {
@@ -675,9 +846,20 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
    // fails, since long lived HTTP connections will
    // go away sometimes and we shouldn't treat this as
    // erroneous. We set the state to 'reconnecting'
-    // instead, so that clients can onserve this state
+    // instead, so that clients can observe this state
    // if they wish.
-    this._startKeepAlives().then(() => {
+    this._startKeepAlives().then((connDidFail) => {
+        // Only emit CATCHUP if we detected a connectivity error: if we didn't,
+        // it's quite likely the sync will fail again for the same reason and we
+        // want to stay in ERROR rather than keep flip-flopping between ERROR
+        // and CATCHUP.
+        if (connDidFail && this.getSyncState() === 'ERROR') {
+            this._updateSyncState("CATCHUP", {
+                oldSyncToken: null,
+                nextSyncToken: null,
+                catchingUp: true,
+            });
+        }
        this._sync(syncOptions);
    });

@@ -686,6 +868,7 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
    this._updateSyncState(
        this._failedSyncCount >= FAILED_SYNC_ERROR_THRESHOLD ?
            "ERROR" : "RECONNECTING",
+        { error: err },
    );
 };

@@ -693,11 +876,12 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
 * Process data returned from a sync response and propagate it
 * into the model objects
 *
- * @param {string} syncToken the old next_batch token sent to this
- *    sync request.
+ * @param {Object} syncEventData Object containing sync tokens associated with this sync
 * @param {Object} data The response from /sync
 */
-SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
+SyncApi.prototype._processSyncResponse = async function(
+    syncEventData, data,
+) {
    const client = this.client;
    const self = this;

@@ -720,6 +904,11 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
    //          state: { events: [] },
    //          timeline: { events: [], prev_batch: $token, limited: true },
    //          ephemeral: { events: [] },
+    //          summary: {
+    //             m.heroes: [ $user_id ],
+    //             m.joined_member_count: $count,
+    //             m.invited_member_count: $count
+    //          },
    //          account_data: { events: [] },
    //          unread_notifications: {
    //              highlight_count: 0,
@@ -776,6 +965,10 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
        client.store.storeAccountDataEvents(events);
        events.forEach(
            function(accountDataEvent) {
+                // Honour push rules that come down the sync stream but also
+                // honour push rules that were previously cached. Base rules
+                // will be updated when we recieve push rules via getPushRules
+                // (see SyncApi.prototype.sync) before syncing over the network.
                if (accountDataEvent.getType() == 'm.push_rules') {
                    client.pushRules = accountDataEvent.getContent();
                }
@@ -854,9 +1047,11 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
        const room = inviteObj.room;
        const stateEvents =
            self._mapSyncEventsFormat(inviteObj.invite_state, room);
+
+        room.updateMyMembership("invite");
        self._processRoomEvents(room, stateEvents);
        if (inviteObj.isBrandNewRoom) {
-            room.recalculate(client.credentials.userId);
+            room.recalculate();
            client.store.storeRoom(room);
            client.emit("Room", room);
        }
@@ -883,6 +1078,8 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
            );
        }

+        room.updateMyMembership("join");
+
        joinObj.timeline = joinObj.timeline || {};

        if (joinObj.isBrandNewRoom) {
@@ -932,7 +1129,8 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
                self._deregisterStateListeners(room);
                room.resetLiveTimeline(
                    joinObj.timeline.prev_batch,
-                    self.opts.canResetEntireTimeline(room.roomId) ? null : syncToken,
+                    self.opts.canResetEntireTimeline(room.roomId) ?
+                        null : syncEventData.oldSyncToken,
                );

                // We have to assume any gap in any timeline is
@@ -946,6 +1144,13 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {

        self._processRoomEvents(room, stateEvents, timelineEvents);

+        // set summary after processing events,
+        // because it will trigger a name calculation
+        // which needs the room state to be up to date
+        if (joinObj.summary) {
+            room.setSummary(joinObj.summary);
+        }
+
        // XXX: should we be adding ephemeralEvents to the timeline?
        // It feels like that for symmetry with room.addAccountData()
        // there should be a room.addEphemeralEvents() or similar.
@@ -954,12 +1159,14 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
        // we deliberately don't add accountData to the timeline
        room.addAccountData(accountDataEvents);

-        room.recalculate(client.credentials.userId);
+        room.recalculate();
        if (joinObj.isBrandNewRoom) {
            client.store.storeRoom(room);
            client.emit("Room", room);
        }

+        self._processEventsForNotifs(room, timelineEvents);
+
        async function processRoomEvent(e) {
            client.emit("event", e);
            if (e.isState() && e.getType() == "m.room.encryption" && self.opts.crypto) {
@@ -987,15 +1194,19 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
        const accountDataEvents =
            self._mapSyncEventsFormat(leaveObj.account_data);

+        room.updateMyMembership("leave");
+
        self._processRoomEvents(room, stateEvents, timelineEvents);
        room.addAccountData(accountDataEvents);

-        room.recalculate(client.credentials.userId);
+        room.recalculate();
        if (leaveObj.isBrandNewRoom) {
            client.store.storeRoom(room);
            client.emit("Room", room);
        }

+        self._processEventsForNotifs(room, timelineEvents);
+
        stateEvents.forEach(function(e) {
            client.emit("event", e);
        });
@@ -1012,7 +1223,7 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
    // in the timeline relative to ones paginated in by /notifications.
    // XXX: we could fix this by making EventTimeline support chronological
    // ordering... but it doesn't, right now.
-    if (syncToken && this._notifEvents.length) {
+    if (syncEventData.oldSyncToken && this._notifEvents.length) {
        this._notifEvents.sort(function(a, b) {
            return a.getTs() - b.getTs();
        });
@@ -1024,7 +1235,9 @@ SyncApi.prototype._processSyncResponse = async function(syncToken, data) {
    // Handle device list updates
    if (data.device_lists) {
        if (this.opts.crypto) {
-            await this.opts.crypto.handleDeviceListChanges(data.device_lists);
+            await this.opts.crypto.handleDeviceListChanges(
+                syncEventData, data.device_lists,
+            );
        } else {
            // FIXME if we *don't* have a crypto module, we still need to
            // invalidate the device lists. But that would require a
@@ -1075,13 +1288,16 @@ SyncApi.prototype._startKeepAlives = function(delay) {
 *
 * On failure, schedules a call back to itself. On success, resolves
 * this._connectionReturnedDefer.
+ *
+ * @param {bool} connDidFail True if a connectivity failure has been detected. Optional.
 */
-SyncApi.prototype._pokeKeepAlive = function() {
+SyncApi.prototype._pokeKeepAlive = function(connDidFail) {
+    if (connDidFail === undefined) connDidFail = false;
    const self = this;
    function success() {
        clearTimeout(self._keepAliveTimer);
        if (self._connectionReturnedDefer) {
-            self._connectionReturnedDefer.resolve();
+            self._connectionReturnedDefer.resolve(connDidFail);
            self._connectionReturnedDefer = null;
        }
    }
@@ -1098,7 +1314,7 @@ SyncApi.prototype._pokeKeepAlive = function() {
    ).done(function() {
        success();
    }, function(err) {
-        if (err.httpStatus == 400) {
+        if (err.httpStatus == 400 || err.httpStatus == 404) {
            // treat this as a success because the server probably just doesn't
            // support /versions: point is, we're getting a response.
            // We wait a short time though, just in case somehow the server
@@ -1106,8 +1322,9 @@ SyncApi.prototype._pokeKeepAlive = function() {
            // responses fail, this will mean we don't hammer in a loop.
            self._keepAliveTimer = setTimeout(success, 2000);
        } else {
+            connDidFail = true;
            self._keepAliveTimer = setTimeout(
-                self._pokeKeepAlive.bind(self),
+                self._pokeKeepAlive.bind(self, connDidFail),
                5000 + Math.floor(Math.random() * 5000),
            );
            // A keepalive has failed, so we emit the
@@ -1115,7 +1332,7 @@ SyncApi.prototype._pokeKeepAlive = function() {
            // first failure).
            // Note we do this after setting the timer:
            // this lets the unit tests advance the mock
-            // clock when the get the error.
+            // clock when they get the error.
            self._updateSyncState("ERROR", { error: err });
        }
    });
@@ -1144,7 +1361,7 @@ SyncApi.prototype._processGroupSyncEntry = function(groupsSection, sectionName)
        }
        group.setMyMembership(sectionName);
        if (isBrandNew) {
-            this.client.store.storeGroup(group);
+            // Now we've filled in all the fields, emit the Group event
            this.client.emit("Group", group);
        }
    }
@@ -1246,50 +1463,82 @@ SyncApi.prototype._resolveInvites = function(room) {
 */
 SyncApi.prototype._processRoomEvents = function(room, stateEventList,
                                                timelineEventList) {
-    timelineEventList = timelineEventList || [];
-    const client = this.client;
-    // "old" and "current" state are the same initially; they
-    // start diverging if the user paginates.
-    // We must deep copy otherwise membership changes in old state
-    // will leak through to current state!
-    const oldStateEvents = utils.map(
-        utils.deepCopy(
-            stateEventList.map(function(mxEvent) {
-                return mxEvent.event;
-            }),
-        ), client.getEventMapper(),
-    );
-    const stateEvents = stateEventList;
-
-    // set the state of the room to as it was before the timeline executes
-    //
-    // XXX: what if we've already seen (some of) the events in the timeline,
-    // and they modify some of the state set in stateEvents? In that case we'll
-    // end up with the state from stateEvents, instead of the more recent state
-    // from the timeline.
-    room.oldState.setStateEvents(oldStateEvents);
-    room.currentState.setStateEvents(stateEvents);
+    // If there are no events in the timeline yet, initialise it with
+    // the given state events
+    const liveTimeline = room.getLiveTimeline();
+    const timelineWasEmpty = liveTimeline.getEvents().length == 0;
+    if (timelineWasEmpty) {
+        // Passing these events into initialiseState will freeze them, so we need
+        // to compute and cache the push actions for them now, otherwise sync dies
+        // with an attempt to assign to read only property.
+        // XXX: This is pretty horrible and is assuming all sorts of behaviour from
+        // these functions that it shouldn't be. We should probably either store the
+        // push actions cache elsewhere so we can freeze MatrixEvents, or otherwise
+        // find some solution where MatrixEvents are immutable but allow for a cache
+        // field.
+        for (const ev of stateEventList) {
+            this.client.getPushActionsForEvent(ev);
+        }
+        liveTimeline.initialiseState(stateEventList);
+    }

    this._resolveInvites(room);

    // recalculate the room name at this point as adding events to the timeline
    // may make notifications appear which should have the right name.
-    room.recalculate(this.client.credentials.userId);
+    // XXX: This looks suspect: we'll end up recalculating the room once here
+    // and then again after adding events (_processSyncResponse calls it after
+    // calling us) even if no state events were added. It also means that if
+    // one of the room events in timelineEventList is something that needs
+    // a recalculation (like m.room.name) we won't recalculate until we've
+    // finished adding all the events, which will cause the notification to have
+    // the old room name rather than the new one.
+    room.recalculate();

+    // If the timeline wasn't empty, we process the state events here: they're
+    // defined as updates to the state before the start of the timeline, so this
+    // starts to roll the state forward.
+    // XXX: That's what we *should* do, but this can happen if we were previously
+    // peeking in a room, in which case we obviously do *not* want to add the
+    // state events here onto the end of the timeline. Historically, the js-sdk
+    // has just set these new state events on the old and new state. This seems
+    // very wrong because there could be events in the timeline that diverge the
+    // state, in which case this is going to leave things out of sync. However,
+    // for now I think it;s best to behave the same as the code has done previously.
+    if (!timelineWasEmpty) {
+        // XXX: As above, don't do this...
+        //room.addLiveEvents(stateEventList || []);
+        // Do this instead...
+        room.oldState.setStateEvents(stateEventList || []);
+        room.currentState.setStateEvents(stateEventList || []);
+    }
+    // execute the timeline events. This will continue to diverge the current state
+    // if the timeline has any state events in it.
+    // This also needs to be done before running push rules on the events as they need
+    // to be decorated with sender etc.
+    room.addLiveEvents(timelineEventList || []);
+};
+
+/**
+ * Takes a list of timelineEvents and adds and adds to _notifEvents
+ * as appropriate.
+ * This must be called after the room the events belong to has been stored.
+ *
+ * @param {Room} room
+ * @param {MatrixEvent[]} [timelineEventList] A list of timeline events. Lower index
+ * is earlier in time. Higher index is later.
+ */
+SyncApi.prototype._processEventsForNotifs = function(room, timelineEventList) {
    // gather our notifications into this._notifEvents
-    if (client.getNotifTimelineSet()) {
+    if (this.client.getNotifTimelineSet()) {
        for (let i = 0; i < timelineEventList.length; i++) {
-            const pushActions = client.getPushActionsForEvent(timelineEventList[i]);
+            const pushActions = this.client.getPushActionsForEvent(timelineEventList[i]);
            if (pushActions && pushActions.notify &&
                pushActions.tweaks && pushActions.tweaks.highlight) {
                this._notifEvents.push(timelineEventList[i]);
            }
        }
    }
-
-    // execute the timeline events, this will begin to diverge the current state
-    // if the timeline has any state events in it.
-    room.addLiveEvents(timelineEventList);
 };

 /**
@@ -1319,6 +1568,7 @@ SyncApi.prototype._getGuestFilter = function() {
 SyncApi.prototype._updateSyncState = function(newState, data) {
    const old = this._syncState;
    this._syncState = newState;
+    this._syncStateData = data;
    this.client.emit("sync", this._syncState, old, data);
 };

@@ -652,3 +652,23 @@ module.exports.inherits = function(ctor, superCtor) {
        },
    });
 };
+
+/**
+ * Returns whether the given value is a finite number without type-coercion
+ *
+ * @param {*} value the value to test
+ * @return {boolean} whether or not value is a finite number without type-coercion
+ */
+module.exports.isNumber = function(value) {
+    return typeof value === 'number' && isFinite(value);
+};
+
+/**
+ * Removes zero width chars, diacritics and whitespace from the string
+ * @param {string} str the string to remove hidden characters from
+ * @return {string} a string with the hidden characters removed
+ */
+module.exports.removeHiddenChars = function(str) {
+    return str.normalize('NFD').replace(removeHiddenCharsRegex, '');
+};
+const removeHiddenCharsRegex = /[\u200B-\u200D\u0300-\u036f\uFEFF\s]/g;
@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,31 +25,6 @@ const DEBUG = true;  // set true to enable console logging.

 // events: hangup, error(err), replaced(call), state(state, oldState)

-/**
- * Fires when the MatrixCall encounters an error when sending a Matrix event.
- * <p>
- * This is required to allow errors, which occur during sending of events, to bubble up.
- * (This is because call.js does a hangup when it encounters a normal `error`, which in
- * turn could lead to an UnknownDeviceError.)
- * <p>
- * To deal with an UnknownDeviceError when trying to send events, the application should let
- * users know that there are new devices in the encrypted room (into which the event was
- * sent) and give the user the options to resend unsent events or cancel them. Resending
- * is done using {@link module:client~MatrixClient#resendEvent} and cancelling can be done by using
- * {@link module:client~MatrixClient#cancelPendingEvent}.
- * <p>
- * MatrixCall will not do anything in response to an error that causes `send_event_error`
- * to be emitted with the exception of sending `m.call.candidates`, which is retried upon
- * failure when ICE candidates are being sent. This happens during call setup.
- *
- * @event module:webrtc/call~MatrixCall#"send_event_error"
- * @param {Error} err The error caught from calling client.sendEvent in call.js.
- * @example
- * matrixCall.on("send_event_error", function(err){
- *   console.error(err);
- * });
- */
-
 /**
 * Fires whenever an error occurs when call.js encounters an issue with setting up the call.
 * <p>
@@ -110,6 +86,8 @@ function MatrixCall(opts) {
    this.mediaPromises = Object.create(null);

    this.screenSharingStream = null;
+
+    this._answerContent = null;
 }
 /** The length of time a call can be ringing for. */
 MatrixCall.CALL_TIMEOUT_MS = 60000;
@@ -123,6 +101,24 @@ MatrixCall.ERR_LOCAL_OFFER_FAILED = "local_offer_failed";
 */
 MatrixCall.ERR_NO_USER_MEDIA = "no_user_media";

+/*
+ * Error code used when a call event failed to send
+ * because unknown devices were present in the room
+ */
+MatrixCall.ERR_UNKNOWN_DEVICES = "unknown_devices";
+
+/*
+ * Error code usewd when we fail to send the invite
+ * for some reason other than there being unknown devices
+ */
+MatrixCall.ERR_SEND_INVITE = "send_invite";
+
+/*
+ * Error code usewd when we fail to send the answer
+ * for some reason other than there being unknown devices
+ */
+MatrixCall.ERR_SEND_ANSWER = "send_answer";
+
 utils.inherits(MatrixCall, EventEmitter);

 /**
@@ -409,6 +405,11 @@ MatrixCall.prototype.answer = function() {
    debuglog("Answering call %s of type %s", this.callId, this.type);
    const self = this;

+    if (self._answerContent) {
+        self._sendAnswer();
+        return;
+    }
+
    if (!this.localAVStream && !this.waitForLocalAVStream) {
        this.webRtc.getUserMedia(
            _getUserMediaVideoContraints(this.type),
@@ -457,6 +458,8 @@ MatrixCall.prototype._replacedBy = function(newCall) {
 * @param {boolean} suppressEvent True to suppress emitting an event.
 */
 MatrixCall.prototype.hangup = function(reason, suppressEvent) {
+    if (this.state == 'ended') return;
+
    debuglog("Ending call " + this.callId);
    terminate(this, "local", reason, !suppressEvent);
    const content = {
@@ -593,6 +596,28 @@ MatrixCall.prototype._maybeGotUserMediaForInvite = function(stream) {
    setState(self, 'create_offer');
 };

+MatrixCall.prototype._sendAnswer = function(stream) {
+    sendEvent(this, 'm.call.answer', this._answerContent).then(() => {
+        setState(this, 'connecting');
+        // If this isn't the first time we've tried to send the answer,
+        // we may have candidates queued up, so send them now.
+        _sendCandidateQueue(this);
+    }).catch((error) => {
+        // We've failed to answer: back to the ringing state
+        setState(this, 'ringing');
+        this.client.cancelPendingEvent(error.event);
+
+        let code = MatrixCall.ERR_SEND_ANSWER;
+        let message = "Failed to send answer";
+        if (error.name == 'UnknownDeviceError') {
+            code = MatrixCall.ERR_UNKNOWN_DEVICES;
+            message = "Unknown devices present in the room";
+        }
+        this.emit("error", callError(code, message));
+        throw error;
+    });
+};
+
 /**
 * Internal
 * @private
@@ -641,7 +666,7 @@ MatrixCall.prototype._maybeGotUserMediaForAnswer = function(stream) {
    self.peerConn.createAnswer(function(description) {
        debuglog("Created answer: " + description);
        self.peerConn.setLocalDescription(description, function() {
-            const content = {
+            self._answerContent = {
                version: 0,
                call_id: self.callId,
                answer: {
@@ -649,8 +674,7 @@ MatrixCall.prototype._maybeGotUserMediaForAnswer = function(stream) {
                    type: self.peerConn.localDescription.type,
                },
            };
-            sendEvent(self, 'm.call.answer', content);
-            setState(self, 'connecting');
+            self._sendAnswer();
        }, function() {
            debuglog("Error setting local description!");
        }, constraints);
@@ -671,6 +695,9 @@ MatrixCall.prototype._gotLocalIceCandidate = function(event) {
            "Got local ICE " + event.candidate.sdpMid + " candidate: " +
            event.candidate.candidate,
        );
+
+        if (this.state == 'ended') return;
+
        // As with the offer, note we need to make a copy of this object, not
        // pass the original: that broke in Chrome ~m43.
        const c = {
@@ -753,14 +780,26 @@ MatrixCall.prototype._gotLocalOffer = function(description) {
            },
            lifetime: MatrixCall.CALL_TIMEOUT_MS,
        };
-        sendEvent(self, 'm.call.invite', content);
-
-        setTimeout(function() {
-            if (self.state == 'invite_sent') {
-                self.hangup('invite_timeout');
+        sendEvent(self, 'm.call.invite', content).then(() => {
+            setState(self, 'invite_sent');
+            setTimeout(function() {
+                if (self.state == 'invite_sent') {
+                    self.hangup('invite_timeout');
+                }
+            }, MatrixCall.CALL_TIMEOUT_MS);
+        }).catch((error) => {
+            let code = MatrixCall.ERR_SEND_INVITE;
+            let message = "Failed to send invite";
+            if (error.name == 'UnknownDeviceError') {
+                code = MatrixCall.ERR_UNKNOWN_DEVICES;
+                message = "Unknown devices present in the room";
            }
-        }, MatrixCall.CALL_TIMEOUT_MS);
-        setState(self, 'invite_sent');
+
+            self.client.cancelPendingEvent(error.event);
+            terminate(self, "local", code, false);
+            self.emit("error", callError(code, message));
+            throw error;
+        });
    }, function() {
        debuglog("Error setting local description!");
    });
@@ -784,6 +823,7 @@ MatrixCall.prototype._getLocalOfferFailed = function(error) {
 * @param {Object} error
 */
 MatrixCall.prototype._getUserMediaFailed = function(error) {
+    terminate(this, "local", 'user_media_failed', false);
    this.emit(
        "error",
        callError(
@@ -792,7 +832,6 @@ MatrixCall.prototype._getUserMediaFailed = function(error) {
            "does this app have permission?",
        ),
    );
-    this.hangup("user_media_failed");
 };

 /**
@@ -972,17 +1011,20 @@ const setState = function(self, state) {
 * @return {Promise}
 */
 const sendEvent = function(self, eventType, content) {
-    return self.client.sendEvent(self.roomId, eventType, content).catch(
-        (err) => {
-            self.emit('send_event_error', err);
-        },
-    );
+    return self.client.sendEvent(self.roomId, eventType, content);
 };

 const sendCandidate = function(self, content) {
    // Sends candidates with are sent in a special way because we try to amalgamate
    // them into one message
    self.candidateSendQueue.push(content);
+
+    // Don't send the ICE candidates yet if the call is in the ringing state: this
+    // means we tried to pick (ie. started generating candidates) and then failed to
+    // send the answer and went back to the ringing state. Queue up the candidates
+    // to send if we sucessfully send the answer.
+    if (self.state == 'ringing') return;
+
    if (self.candidateSendTries === 0) {
        setTimeout(function() {
            _sendCandidateQueue(self);
@@ -1079,9 +1121,13 @@ const _tryPlayRemoteStream = function(self) {
    }
 };

-const _tryPlayRemoteAudioStream = function(self) {
+const _tryPlayRemoteAudioStream = async function(self) {
    if (self.getRemoteAudioElement() && self.remoteAStream) {
        const player = self.getRemoteAudioElement();
+
+        // if audioOutput is non-default:
+        if (audioOutput) await player.setSinkId(audioOutput);
+
        player.autoplay = true;
        self.assignElement(player, self.remoteAStream, "remoteAudio");
        setTimeout(function() {
@@ -1275,8 +1321,16 @@ const forAllTracksOnStream = function(s, f) {
 /** The MatrixCall class. */
 module.exports.MatrixCall = MatrixCall;

+let audioOutput;
 let audioInput;
 let videoInput;
+/**
+ * Set an audio output device to use for MatrixCalls
+ * @function
+ * @param {string=} deviceId the identifier for the device
+ * undefined treated as unset
+ */
+module.exports.setAudioOutput = function(deviceId) { audioOutput = deviceId; };
 /**
 * Set an audio input device to use for MatrixCalls
 * @function
@@ -1296,8 +1350,8 @@ module.exports.setVideoInput = function(deviceId) { videoInput = deviceId; };
 * Create a new Matrix call for the browser.
 * @param {MatrixClient} client The client instance to use.
 * @param {string} roomId The room the call is in.
- * @param {Object?} options optional options map.
- * @param {boolean} options.forceTURN whether relay through TURN should be forced.
+ * @param {Object?} options DEPRECATED optional options map.
+ * @param {boolean} options.forceTURN DEPRECATED whether relay through TURN should be forced. This option is deprecated - use opts.forceTURN when creating the matrix client since it's only possible to set this option on outbound calls.
 * @return {MatrixCall} the call or null if the browser doesn't support calling.
 */
 module.exports.createNewMatrixCall = function(client, roomId, options) {
@@ -1350,6 +1404,9 @@ module.exports.createNewMatrixCall = function(client, roomId, options) {
            !webRtc.RtcPeerConnection || !webRtc.getUserMedia) {
        return null; // WebRTC is not supported.
    }
+
+    const optionsForceTURN = options ? options.forceTURN : false;
+
    const opts = {
        webRtc: webRtc,
        client: client,
@@ -1357,7 +1414,7 @@ module.exports.createNewMatrixCall = function(client, roomId, options) {
        roomId: roomId,
        turnServers: client.getTurnServers(),
        // call level options
-        forceTURN: options ? options.forceTURN : false,
+        forceTURN: client._forceTURN || optionsForceTURN,
    };
    return new MatrixCall(opts);
 };
@@ -5,7 +5,7 @@ set -ex
 npm run lint

 # install Olm so that we can run the crypto tests.
-npm install https://matrix.org/packages/npm/olm/olm-2.2.2.tgz
+npm install https://matrix.org/packages/npm/olm/olm-3.0.0.tgz

 npm run test