v0.7.1-rc.1

Prepare changelog for v0.7.1-rc.1
Merge pull request #300 from matrix-org/rav/handle_no_session_store
2016-12-05 17:43:39 +00:00 · 2016-12-05 17:43:39 +00:00 · 2016-11-30 11:06:17 +00:00 · 2016-11-30 10:48:40 +00:00 · 2016-11-24 14:55:48 +00:00 · 2016-11-24 14:51:35 +00:00
66 changed files with 3305 additions and 298652 deletions
@@ -1,5 +1,3 @@
-# Keep this file in sync with .npmignore.
-
 .jsdoc
 node_modules
 .lock-wscript
@@ -8,7 +6,7 @@ coverage
 lib-cov
 out
 reports
-dist/browser-matrix-dev.js
+/dist

 # version file and tarball created by 'npm pack'
 /git-revision.txt
@@ -1,15 +0,0 @@
-# Keep this file in sync with .gitignore.
-
-.jsdoc
-node_modules
-.lock-wscript
-build/Release
-coverage
-lib-cov
-out
-reports
-dist/browser-matrix-dev.js
-
-# tarball created by 'npm pack'.
-/matrix-js-sdk-*.tgz
-
@@ -0,0 +1,3 @@
+language: node_js
+node_js:
+    - node # Latest stable version of nodejs.
@@ -1,3 +1,148 @@
+Changes in [0.7.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.1-rc.1) (2016-12-05)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.7.0...v0.7.1-rc.1)
+
+ * Avoid NPE when no sessionStore is given
+   [\#300](https://github.com/matrix-org/matrix-js-sdk/pull/300)
+ * Improve decryption error messages
+   [\#299](https://github.com/matrix-org/matrix-js-sdk/pull/299)
+ * Revert "Use native Array.isArray when available."
+   [\#283](https://github.com/matrix-org/matrix-js-sdk/pull/283)
+ * Use native Array.isArray when available.
+   [\#282](https://github.com/matrix-org/matrix-js-sdk/pull/282)
+
+Changes in [0.7.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.7.0) (2016-11-18)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.4...v0.7.0)
+
+ * Avoid a packetstorm of device queries on startup
+   [\#297](https://github.com/matrix-org/matrix-js-sdk/pull/297)
+ * E2E: Check devices to share keys with on each send
+   [\#295](https://github.com/matrix-org/matrix-js-sdk/pull/295)
+ * Apply unknown-keyshare mitigations
+   [\#296](https://github.com/matrix-org/matrix-js-sdk/pull/296)
+ * distinguish unknown users from deviceless users
+   [\#294](https://github.com/matrix-org/matrix-js-sdk/pull/294)
+ * Allow starting client with initialSyncLimit = 0
+   [\#293](https://github.com/matrix-org/matrix-js-sdk/pull/293)
+ * Make timeline-window _unpaginate public and rename to unpaginate
+   [\#289](https://github.com/matrix-org/matrix-js-sdk/pull/289)
+ * Send a STOPPED sync updated after call to stopClient
+   [\#286](https://github.com/matrix-org/matrix-js-sdk/pull/286)
+ * Fix bug in verifying megolm event senders
+   [\#292](https://github.com/matrix-org/matrix-js-sdk/pull/292)
+ * Handle decryption of events after they arrive
+   [\#288](https://github.com/matrix-org/matrix-js-sdk/pull/288)
+ * Fix examples.
+   [\#287](https://github.com/matrix-org/matrix-js-sdk/pull/287)
+ * Add a travis.yml
+   [\#278](https://github.com/matrix-org/matrix-js-sdk/pull/278)
+ * Encrypt all events, including 'm.call.*'
+   [\#277](https://github.com/matrix-org/matrix-js-sdk/pull/277)
+ * Ignore reshares of known megolm sessions
+   [\#276](https://github.com/matrix-org/matrix-js-sdk/pull/276)
+ * Log to the console on unknown session
+   [\#274](https://github.com/matrix-org/matrix-js-sdk/pull/274)
+ * Make it easier for SDK users to wrap prevailing the 'request' function
+   [\#273](https://github.com/matrix-org/matrix-js-sdk/pull/273)
+
+Changes in [0.6.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.4) (2016-11-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.4-rc.2...v0.6.4)
+
+ * Change release script to pass version by environment variable
+
+
+Changes in [0.6.4-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.4-rc.2) (2016-11-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.4-rc.1...v0.6.4-rc.2)
+
+ * Add getRoomTags method to client
+   [\#236](https://github.com/matrix-org/matrix-js-sdk/pull/236)
+
+Changes in [0.6.4-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.4-rc.1) (2016-11-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.3...v0.6.4-rc.1)
+
+Breaking Changes
+----------------
+ * Bundled version of the JS SDK are no longer versioned along with
+   source files in the dist/ directory. As of this release, they
+   will be included in the release tarball, but not the source
+   repository.
+
+Other Changes
+-------------
+ * More fixes to the release script
+   [\#272](https://github.com/matrix-org/matrix-js-sdk/pull/272)
+ * Update the release process to use github releases
+   [\#271](https://github.com/matrix-org/matrix-js-sdk/pull/271)
+ * Don't package the world when we release
+   [\#270](https://github.com/matrix-org/matrix-js-sdk/pull/270)
+ * Add ability to set a filter prior to the first /sync
+   [\#269](https://github.com/matrix-org/matrix-js-sdk/pull/269)
+ * Sign one-time keys, and verify their signatures
+   [\#243](https://github.com/matrix-org/matrix-js-sdk/pull/243)
+ * Check for duplicate message indexes for group messages
+   [\#241](https://github.com/matrix-org/matrix-js-sdk/pull/241)
+ * Rotate megolm sessions
+   [\#240](https://github.com/matrix-org/matrix-js-sdk/pull/240)
+ * Check recipient and sender in Olm messages
+   [\#239](https://github.com/matrix-org/matrix-js-sdk/pull/239)
+ * Consistency checks for E2E device downloads
+   [\#237](https://github.com/matrix-org/matrix-js-sdk/pull/237)
+ * Support User-Interactive auth for delete device
+   [\#235](https://github.com/matrix-org/matrix-js-sdk/pull/235)
+ * Utility to help with interactive auth
+   [\#234](https://github.com/matrix-org/matrix-js-sdk/pull/234)
+ * Fix sync breaking when an invalid filterId is in localStorage
+   [\#228](https://github.com/matrix-org/matrix-js-sdk/pull/228)
+
+Changes in [0.6.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.3) (2016-10-12)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.2...v0.6.3)
+
+Breaking Changes
+----------------
+ * Add a 'RECONNECTING' state to the sync states. This is an additional state
+   between 'SYNCING' and 'ERROR', so most clients should not notice.
+
+Other Changes
+----------------
+ * Fix params getting replaced on register calls
+   [\#233](https://github.com/matrix-org/matrix-js-sdk/pull/233)
+ * Fix potential 30s delay on reconnect
+   [\#232](https://github.com/matrix-org/matrix-js-sdk/pull/232)
+ * uploadContent: Attempt some consistency between browser and node
+   [\#230](https://github.com/matrix-org/matrix-js-sdk/pull/230)
+ * Fix error handling on uploadContent
+   [\#229](https://github.com/matrix-org/matrix-js-sdk/pull/229)
+ * Fix uploadContent for node.js
+   [\#226](https://github.com/matrix-org/matrix-js-sdk/pull/226)
+ * Don't emit ERROR until a keepalive poke fails
+   [\#223](https://github.com/matrix-org/matrix-js-sdk/pull/223)
+ * Function to get the fallback url for interactive auth
+   [\#224](https://github.com/matrix-org/matrix-js-sdk/pull/224)
+ * Revert "Handle the first /sync failure differently."
+   [\#222](https://github.com/matrix-org/matrix-js-sdk/pull/222)
+
+Changes in [0.6.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.2) (2016-10-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.1...v0.6.2)
+
+ * Check dependencies aren't on develop in release.sh
+   [\#221](https://github.com/matrix-org/matrix-js-sdk/pull/221)
+ * Fix checkTurnServers leak on logout
+   [\#220](https://github.com/matrix-org/matrix-js-sdk/pull/220)
+ * Fix leak of file upload objects
+   [\#219](https://github.com/matrix-org/matrix-js-sdk/pull/219)
+ * crypto: remove duplicate code
+   [\#218](https://github.com/matrix-org/matrix-js-sdk/pull/218)
+ * Add API for 3rd party location lookup
+   [\#217](https://github.com/matrix-org/matrix-js-sdk/pull/217)
+ * Handle the first /sync failure differently.
+   [\#216](https://github.com/matrix-org/matrix-js-sdk/pull/216)
+
 Changes in [0.6.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v0.6.1) (2016-09-21)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v0.6.0...v0.6.1)
@@ -0,0 +1,4 @@
+Contributing code to matrix-js-sdk
+==================================
+
+matrix-js-sdk follows the same pattern as https://github.com/matrix-org/synapse/blob/master/CONTRIBUTING.rst
@@ -10,9 +10,10 @@ Quickstart

 In a browser
 ------------
-Copy ``dist/$VERSION/browser-matrix-$VERSION.js`` and add that as a ``<script>`` to
-your page. There will be a global variable ``matrixcs`` attached to
-``window`` through which you can access the SDK.
+Download either the full or minified version from
+https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
+``<script>`` to your page. There will be a global variable ``matrixcs``
+attached to ``window`` through which you can access the SDK.

 Please check [the working browser example](examples/browser) for more information.

@@ -1 +0,0 @@
-Release builds and development builds will reside here.
@@ -1 +1 @@
-../../../dist/browser-matrix-dev.js
+../../../dist/browser-matrix.js
@@ -135,11 +135,15 @@ rl.on('line', function(line) {
 // ==== END User input

 // show the room list after syncing.
-matrixClient.on("syncComplete", function() {
-    setRoomList();
-    printRoomList();
-    printHelp();
-    rl.prompt();
+matrixClient.on("sync", function(state, prevState, data) {
+    switch (state) {
+        case "PREPARED":
+          setRoomList();
+          printRoomList();
+          printHelp();
+          rl.prompt();
+        break;
+   }
 });

 matrixClient.on("Room", function() {
@@ -44,7 +44,15 @@ window.onload = function() {
    disableButtons(true, true, true);
 };

-client.on("syncComplete", function () {
+matrixClient.on("sync", function(state, prevState, data) {
+    switch (state) {
+        case "PREPARED":
+          syncComplete();
+        break;
+   }
+});
+
+function syncComplete() {
    document.getElementById("result").innerHTML = "<p>Ready for calls.</p>";
    disableButtons(false, true, true);

@@ -85,5 +93,5 @@ client.on("syncComplete", function () {
        call = c;
        addListeners(call);
    });
-});
+}
 client.startClient();
@@ -1 +1 @@
-../../../dist/browser-matrix-dev.js
+../../../dist/browser-matrix.js
@@ -28,4 +28,6 @@ rm -f matrix-js-sdk-*.tgz
 npm pack ||
    fail "npm pack finished with return code $?"

+npm run gendoc || fail "JSDoc failed with code $?"
+
 exit $RC
@@ -136,10 +136,12 @@ MatrixBaseApis.prototype.register = function(
    var params = {
        auth: auth
    };
-    if (username !== undefined) { params.username = username; }
-    if (password !== undefined) { params.password = password; }
-    if (bindEmail !== undefined) { params.bind_email = bindEmail; }
-    if (guestAccessToken !== undefined) { params.guest_access_token = guestAccessToken; }
+    if (username !== undefined && username !== null) { params.username = username; }
+    if (password !== undefined && password !== null) { params.password = password; }
+    if (bindEmail !== undefined && bindEmail !== null) { params.bind_email = bindEmail; }
+    if (guestAccessToken !== undefined && guestAccessToken !== null) {
+        params.guest_access_token = guestAccessToken;
+    }

    return this.registerRequest(params, undefined, callback);
 };
@@ -290,6 +292,23 @@ MatrixBaseApis.prototype.deactivateAccount = function(auth, callback) {
    );
 };

+/**
+ * Get the fallback URL to use for unknown interactive-auth stages.
+ *
+ * @param {string} loginType     the type of stage being attempted
+ * @param {string} authSessionId the auth session ID provided by the homeserver
+ *
+ * @return {string} HS URL to hit to for the fallback interface
+ */
+MatrixBaseApis.prototype.getFallbackAuthUrl = function(loginType, authSessionId) {
+    var path = utils.encodeUri("/auth/$loginType/fallback/web", {
+        $loginType: loginType,
+    });
+
+    return this._http.getUrl(path, {
+        session: authSessionId,
+    }, httpApi.PREFIX_R0);
+};

 // Room operations
 // ===============
@@ -557,13 +576,38 @@ MatrixBaseApis.prototype.setRoomDirectoryVisibility =

 /**
 * Upload a file to the media repository on the home server.
- * @param {File} file object
- * @param {module:client.callback} callback Optional.
- * @return {module:client.Promise} Resolves: TODO
- * @return {module:http-api.MatrixError} Rejects: with an error response.
+ *
+ * @param {object} file The object to upload. On a browser, something that
+ *   can be sent to XMLHttpRequest.send (typically a File).  Under node.js,
+ *   a a Buffer, String or ReadStream.
+ *
+ * @param {object} opts  options object
+ *
+ * @param {string=} opts.name   Name to give the file on the server. Defaults
+ *   to <tt>file.name</tt>.
+ *
+ * @param {string=} opts.type   Content-type for the upload. Defaults to
+ *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
+ *
+ * @param {boolean=} opts.rawResponse Return the raw body, rather than
+ *   parsing the JSON. Defaults to false (except on node.js, where it
+ *   defaults to true for backwards compatibility).
+ *
+ * @param {boolean=} opts.onlyContentUri Just return the content URI,
+ *   rather than the whole body. Defaults to false (except on browsers,
+ *   where it defaults to true for backwards compatibility). Ignored if
+ *   opts.rawResponse is true.
+ *
+ * @param {Function=} opts.callback Deprecated. Optional. The callback to
+ *    invoke on success/failure. See the promise return values for more
+ *    information.
+ *
+ * @return {module:client.Promise} Resolves to response object, as
+ *    determined by this.opts.onlyData, opts.rawResponse, and
+ *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
 */
-MatrixBaseApis.prototype.uploadContent = function(file, callback) {
-    return this._http.uploadContent(file, callback);
+MatrixBaseApis.prototype.uploadContent = function(file, opts) {
+    return this._http.uploadContent(file, opts);
 };

 /**
@@ -705,16 +749,23 @@ MatrixBaseApis.prototype.setDeviceDetails = function(device_id, body) {
 * Delete the given device
 *
 * @param {string} device_id  device to delete
+ * @param {object} auth Optional. Auth data to supply for User-Interactive auth.
 * @return {module:client.Promise} Resolves: result object
 * @return {module:http-api.MatrixError} Rejects: with an error response.
 */
-MatrixBaseApis.prototype.deleteDevice = function(device_id) {
+MatrixBaseApis.prototype.deleteDevice = function(device_id, auth) {
    var path = utils.encodeUri("/devices/$device_id", {
        $device_id: device_id,
    });

+    var body = {};
+
+    if (auth) {
+        body.auth = auth;
+    }
+
    return this._http.authedRequestWithPrefix(
-        undefined, "DELETE", path, undefined, undefined,
+        undefined, "DELETE", path, undefined, body,
        httpApi.PREFIX_UNSTABLE
    );
 };
@@ -925,24 +976,28 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, callback) {
 *
 * @param {string[][]} devices  a list of [userId, deviceId] pairs
 *
- * @param {module:client.callback=} callback
+ * @param {string} [key_algorithm = signed_curve25519]  desired key type
 *
 * @return {module:client.Promise} Resolves: result object. Rejects: with
 *     an error response ({@link module:http-api.MatrixError}).
 */
-MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, callback) {
+MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm) {
    var queries = {};

+    if (key_algorithm === undefined) {
+        key_algorithm = "signed_curve25519";
+    }
+
    for (var i = 0; i < devices.length; ++i) {
        var userId = devices[i][0];
        var deviceId = devices[i][1];
        var query = queries[userId] || {};
        queries[userId] = query;
-        query[deviceId] = "curve25519";
+        query[deviceId] = key_algorithm;
    }
    var content = {one_time_keys: queries};
    return this._http.authedRequestWithPrefix(
-        callback, "POST", "/keys/claim", undefined, content,
+        undefined, "POST", "/keys/claim", undefined, content,
        httpApi.PREFIX_UNSTABLE
    );
 };
@@ -1039,6 +1094,39 @@ MatrixBaseApis.prototype.sendToDevice = function(
    );
 };

+// Third party Lookup API
+// ======================
+
+/**
+ * Get the third party protocols that can be reached using
+ * this HS
+ * @return {module:client.Promise} Resolves to the result object
+ */
+MatrixBaseApis.prototype.getThirdpartyProtocols = function() {
+    return this._http.authedRequestWithPrefix(
+        undefined, "GET", "/thirdparty/protocols", undefined, undefined,
+        httpApi.PREFIX_UNSTABLE
+    );
+};
+
+/**
+ * Get information on how a specific place on a third party protocol
+ * may be reached.
+ * @param {string} protocol The protocol given in getThirdpartyProtocols()
+ * @param {object} params Protocol-specific parameters, as given in th
+ *                        response to getThirdpartyProtocols()
+ * @return {module:client.Promise} Resolves to the result object
+ */
+MatrixBaseApis.prototype.getThirdpartyLocation = function(protocol, params) {
+    var path = utils.encodeUri("/thirdparty/location/$protocol", {
+        $protocol: protocol
+    });
+
+    return this._http.authedRequestWithPrefix(
+        undefined, "GET", path, params, undefined,
+        httpApi.PREFIX_UNSTABLE
+    );
+};

 /**
 * MatrixBaseApis object
@@ -149,7 +149,7 @@ function MatrixClient(opts) {
    this._notifTimelineSet = null;

    this._crypto = null;
-    if (CRYPTO_ENABLED && opts.sessionStore !== null &&
+    if (CRYPTO_ENABLED && Boolean(opts.sessionStore) &&
            userId !== null && this.deviceId !== null) {
        this._crypto = new Crypto(
            this, this,
@@ -351,7 +351,7 @@ MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
-    return this._crypto.getStoredDevicesForUser(userId);
+    return this._crypto.getStoredDevicesForUser(userId) || [];
 };


@@ -463,38 +463,31 @@ MatrixClient.prototype.isRoomEncrypted = function(roomId) {
 * Decrypt a received event according to the algorithm specified in the event.
 *
 * @param {MatrixClient} client
- * @param {object} raw event
- *
- * @return {MatrixEvent}
+ * @param {MatrixEvent} event
 */
 function _decryptEvent(client, event) {
    if (!client._crypto) {
-        return _badEncryptedMessage(event, "**Encryption not enabled**");
+        _badEncryptedMessage(event, "Encryption not enabled");
+        return;
    }

-    var decryptionResult;
    try {
-        decryptionResult = client._crypto.decryptEvent(event);
+        client._crypto.decryptEvent(event);
    } catch (e) {
        if (!(e instanceof Crypto.DecryptionError)) {
            throw e;
        }
-        return _badEncryptedMessage(event, "**" + e.message + "**");
+        _badEncryptedMessage(event, e.message);
+        return;
    }
-    return new MatrixEvent(
-        event, decryptionResult.payload,
-        decryptionResult.keysProved,
-        decryptionResult.keysClaimed
-    );
 }

 function _badEncryptedMessage(event, reason) {
-    return new MatrixEvent(event, {
+    event.setClearData({
        type: "m.room.message",
        content: {
            msgtype: "m.bad.encrypted",
-            body: reason,
-            content: event.content,
+            body: "** Unable to decrypt: " + reason + " **",
        },
    });
 }
@@ -696,6 +689,22 @@ MatrixClient.prototype.setRoomTopic = function(roomId, topic, callback) {
                               undefined, callback);
 };

+/**
+ * @param {string} roomId
+ * @param {module:client.callback} callback Optional.
+ * @return {module:client.Promise} Resolves: TODO
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixClient.prototype.getRoomTags = function(roomId, callback) {
+    var path = utils.encodeUri("/user/$userId/rooms/$roomId/tags/", {
+        $userId: this.credentials.userId,
+        $roomId: roomId,
+    });
+    return this._http.authedRequest(
+        callback, "GET", path, undefined
+    );
+};
+
 /**
 * @param {string} roomId
 * @param {string} tagName name of room tag to be set
@@ -2395,7 +2404,26 @@ MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
            }
            // debuglog("Existing filter ID %s: %s; new filter: %s",
            //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
-            return;
+            self.store.setFilterIdByName(filterName, undefined);
+            return undefined;
+        }, function(error) {
+            // Synapse currently returns the following when the filter cannot be found:
+            // {
+            //     errcode: "M_UNKNOWN",
+            //     name: "M_UNKNOWN",
+            //     message: "No row found",
+            //     data: Object, httpStatus: 404
+            // }
+            if (error.httpStatus === 404 &&
+                (error.errcode === "M_UNKNOWN" || error.errcode === "M_NOT_FOUND")) {
+                // Clear existing filterId from localStorage
+                // if it no longer exists on the server
+                self.store.setFilterIdByName(filterName, undefined);
+                // Return a undefined value for existingId further down the promise chain
+                return undefined;
+            } else {
+                throw error;
+            }
        });
    }

@@ -2492,6 +2520,9 @@ MatrixClient.prototype.getTurnServers = function() {
 *
 * @param {Number=} opts.pollTimeout The number of milliseconds to wait on /events.
 * Default: 30000 (30 seconds).
+ *
+ * @param {Filter=} opts.filter The filter to apply to /sync calls. This will override
+ * the opts.initialSyncLimit, which would normally result in a timeline limit filter.
 */
 MatrixClient.prototype.startClient = function(opts) {
    if (this.clientRunning) {
@@ -2543,6 +2574,7 @@ MatrixClient.prototype.stopClient = function() {
    if (this._crypto) {
        global.clearInterval(this._uploadIntervalID);
    }
+    global.clearTimeout(this._checkTurnServersTimeoutID);
 };

 function setupCallEventHandler(client) {
@@ -2762,13 +2794,15 @@ function checkTurnServers(client) {
            };
            client._turnServers = [servers];
            // re-fetch when we're about to reach the TTL
-            setTimeout(function() { checkTurnServers(client); },
-                (res.ttl || (60 * 60)) * 1000 * 0.9
-            );
+            client._checkTurnServersTimeoutID =
+                setTimeout(function() { checkTurnServers(client); },
+                           (res.ttl || (60 * 60)) * 1000 * 0.9
+                          );
        }
    }, function(err) {
        console.error("Failed to get TURN URIs");
-        setTimeout(function() { checkTurnServers(client); }, 60000);
+        client._checkTurnServersTimeoutID =
+            setTimeout(function() { checkTurnServers(client); }, 60000);
    });
 }

@@ -2788,10 +2822,11 @@ function _resolve(callback, defer, res) {

 function _PojoToMatrixEventMapper(client) {
    function mapper(plainOldJsObject) {
-        if (plainOldJsObject.type === "m.room.encrypted") {
-            return _decryptEvent(client, plainOldJsObject);
+        var event = new MatrixEvent(plainOldJsObject);
+        if (event.isEncrypted()) {
+            _decryptEvent(client, event);
        }
-        return new MatrixEvent(plainOldJsObject);
+        return event;
    }
    return mapper;
 }
@@ -2831,6 +2866,10 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;

 /**
 * Fires whenever the SDK receives a new event.
+ * <p>
+ * This is only fired for live events received via /sync - it is not fired for
+ * events received over context, search, or pagination APIs.
+ *
 * @event module:client~MatrixClient#"event"
 * @param {MatrixEvent} event The matrix event which caused this event to fire.
 * @example
@@ -2864,6 +2903,9 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 * the server for updates. This may be called multiple times even if the state is
 * already ERROR. <i>This is the equivalent of "syncError" in the previous
 * API.</i></li>
+ * <li>RECONNECTING: The sync connedtion has dropped, but not in a way that should
+ * be considered erroneous.
+ * </li>
 * <li>STOPPED: The client has stopped syncing with server due to stopClient
 * being called.
 * </li>
@@ -2873,8 +2915,10 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED;
 *                                          +---->STOPPED
 *                                          |
 *              +----->PREPARED -------> SYNCING <--+
- *              |        ^                  |       |
- *   null ------+        |  +---------------+       |
+ *              |        ^                  ^       |
+ *              |        |                  |       |
+ *              |        |                  V       |
+ *   null ------+        |  +-RECONNECTING<-+       |
 *              |        |  V                       |
 *              +------->ERROR ---------------------+
 *
@@ -58,6 +58,18 @@ function OlmDevice(sessionStore) {
    // we don't bother stashing outboundgroupsessions in the sessionstore -
    // instead we keep them here.
    this._outboundGroupSessionStore = {};
+
+    // Store a set of decrypted message indexes for each group session.
+    // This partially mitigates a replay attack where a MITM resends a group
+    // message into the room.
+    //
+    // TODO: If we ever remove an event from memory we will also need to remove
+    // it from this map. Otherwise if we download the event from the server we
+    // will think that it is a duplicate.
+    //
+    // Keys are strings of form "<senderKey>|<session_id>|<message_index>"
+    // Values are true.
+    this._inboundGroupSessionMessageIndexes = {};
 }

 function _initialise_account(sessionStore, pickleKey, account) {
@@ -374,6 +386,10 @@ OlmDevice.prototype.encryptMessage = function(
 ) {
    var self = this;

+    if (payloadString === undefined) {
+        throw new Error("payloadString undefined");
+    }
+
    return this._getSession(theirDeviceIdentityKey, sessionId, function(session) {
        var res = session.encrypt(payloadString);
        self._saveSession(theirDeviceIdentityKey, session);
@@ -611,6 +627,24 @@ OlmDevice.prototype.addInboundGroupSession = function(
    roomId, senderKey, sessionId, sessionKey, keysClaimed
 ) {
    var self = this;
+
+    /* if we already have this session, consider updating it */
+    function updateSession(session) {
+        console.log("Update for megolm session " + senderKey + "/" + sessionId);
+        // for now we just ignore updates. TODO: implement something here
+
+        return true;
+    }
+
+    var r = this._getInboundGroupSession(
+        roomId, senderKey, sessionId, updateSession
+    );
+
+    if (r !== null) {
+        return;
+    }
+
+    // new session.
    var session = new Olm.InboundGroupSession();
    try {
        session.create(sessionKey);
@@ -648,6 +682,22 @@ OlmDevice.prototype.decryptGroupMessage = function(
    function decrypt(session, keysClaimed) {
        var res = session.decrypt(body);

+        var plaintext = res.plaintext;
+        if (plaintext === undefined) {
+            // Compatibility for older olm versions.
+            plaintext = res;
+        } else {
+            // Check if we have seen this message index before to detect replay attacks.
+            var messageIndexKey = senderKey + "|" + sessionId + "|" + res.message_index;
+            if (messageIndexKey in self._inboundGroupSessionMessageIndexes) {
+                throw new Error(
+                    "Duplicate message index, possible replay attack: " +
+                    messageIndexKey
+                );
+            }
+            self._inboundGroupSessionMessageIndexes[messageIndexKey] = true;
+        }
+
        // the sender must have had the senderKey to persuade us to save the
        // session.
        var keysProved = {curve25519: senderKey};
@@ -656,7 +706,7 @@ OlmDevice.prototype.decryptGroupMessage = function(
            roomId, senderKey, sessionId, session, keysClaimed
        );
        return {
-            result: res,
+            result: plaintext,
            keysClaimed: keysClaimed,
            keysProved: keysProved,
        };
@@ -45,13 +45,16 @@ module.exports.DECRYPTION_CLASSES = {};
 * @alias module:crypto/algorithms/base.EncryptionAlgorithm
 *
 * @param {object} params parameters
+ * @param {string} params.userId  The UserID for the local user
 * @param {string} params.deviceId The identifier for this device.
 * @param {module:crypto} params.crypto crypto core
 * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
 * @param {module:base-apis~MatrixBaseApis} baseApis base matrix api interface
 * @param {string} params.roomId  The ID of the room we will be sending to
+ * @param {object} params.config  The body of the m.room.encryption event
 */
 var EncryptionAlgorithm = function(params) {
+    this._userId = params.userId;
    this._deviceId = params.deviceId;
    this._crypto = params.crypto;
    this._olmDevice = params.olmDevice;
@@ -85,15 +88,6 @@ EncryptionAlgorithm.prototype.onRoomMembership = function(
    event, member, oldMembership
 ) {};

-/**
- * Called when a new device announces itself in the room
- *
- * @param {string} userId    owner of the device
- * @param {string} deviceId  deviceId of the device
- */
-EncryptionAlgorithm.prototype.onNewDevice = function(userId, deviceId) {};
-
-
 /**
 * base type for decryption implementations
 *
@@ -101,10 +95,17 @@ EncryptionAlgorithm.prototype.onNewDevice = function(userId, deviceId) {};
 * @alias module:crypto/algorithms/base.DecryptionAlgorithm
 *
 * @param {object} params parameters
+ * @param {string} params.userId  The UserID for the local user
+ * @param {module:crypto} params.crypto crypto core
 * @param {module:crypto/OlmDevice} params.olmDevice olm.js wrapper
+ * @param {string=} params.roomId The ID of the room we will be receiving
+ *     from. Null for to-device events.
 */
 var DecryptionAlgorithm = function(params) {
+    this._userId = params.userId;
+    this._crypto = params.crypto;
    this._olmDevice = params.olmDevice;
+    this._roomId = params.roomId;
 };
 /** */
 module.exports.DecryptionAlgorithm = DecryptionAlgorithm;
@@ -27,6 +27,57 @@ var utils = require("../../utils");
 var olmlib = require("../olmlib");
 var base = require("./base");

+/**
+ * @private
+ * @constructor
+ *
+ * @param {string} sessionId
+ *
+ * @property {string} sessionId
+ * @property {Number} useCount     number of times this session has been used
+ * @property {Number} creationTime when the session was created (ms since the epoch)
+ * @property {module:client.Promise?} sharePromise  If a share operation is in progress,
+ *    a promise which resolves when it is complete.
+ *
+ * @property {object} sharedWithDevices
+ *    devices with which we have shared the session key
+ *        userId -> {deviceId -> msgindex}
+ */
+function OutboundSessionInfo(sessionId) {
+    this.sessionId = sessionId;
+    this.useCount = 0;
+    this.creationTime = new Date().getTime();
+    this.sharePromise = null;
+    this.sharedWithDevices = {};
+}
+
+
+/**
+ * Check if it's time to rotate the session
+ *
+ * @param {Number} rotationPeriodMsgs
+ * @param {Number} rotationPeriodMs
+ * @return {Boolean}
+ */
+OutboundSessionInfo.prototype.needsRotation = function(
+    rotationPeriodMsgs, rotationPeriodMs
+) {
+    var sessionLifetime = new Date().getTime() - this.creationTime;
+
+    if (this.useCount >= rotationPeriodMsgs ||
+        sessionLifetime >= rotationPeriodMs
+       ) {
+        console.log(
+            "Rotating megolm session after " + this.useCount +
+                " messages, " + sessionLifetime + "ms"
+        );
+        return true;
+    }
+
+    return false;
+};
+
+
 /**
 * Megolm encryption implementation
 *
@@ -38,15 +89,23 @@ var base = require("./base");
 */
 function MegolmEncryption(params) {
    base.EncryptionAlgorithm.call(this, params);
-    this._prepPromise = null;
-    this._outboundSessionId = null;
-    this._discardNewSession = false;

-    // devices which have joined since we last sent a message.
-    // userId -> {deviceId -> true}, or
-    // userId -> true
-    this._devicesPendingKeyShare = {};
-    this._sharePromise = null;
+    // OutboundSessionInfo. Null if we haven't yet started setting one up. Note
+    // that even if this is non-null, it may not be ready for use (in which
+    // case _outboundSession.sharePromise will be non-null.)
+    this._outboundSession = null;
+
+    // default rotation periods
+    this._sessionRotationPeriodMsgs = 100;
+    this._sessionRotationPeriodMs = 7 * 24 * 3600 * 1000;
+
+    if (params.config.rotation_period_ms !== undefined) {
+        this._sessionRotationPeriodMs = params.config.rotation_period_ms;
+    }
+
+    if (params.config.rotation_period_msgs !== undefined) {
+        this._sessionRotationPeriodMsgs = params.config.rotation_period_msgs;
+    }
 }
 utils.inherits(MegolmEncryption, base.EncryptionAlgorithm);

@@ -55,59 +114,75 @@ utils.inherits(MegolmEncryption, base.EncryptionAlgorithm);
 *
 * @param {module:models/room} room
 *
- * @return {module:client.Promise} Promise which resolves to the megolm
- *   sessionId when setup is complete.
+ * @return {module:client.Promise} Promise which resolves to the
+ *    OutboundSessionInfo when setup is complete.
 */
 MegolmEncryption.prototype._ensureOutboundSession = function(room) {
    var self = this;

-    if (this._prepPromise) {
-        // prep already in progress
-        return this._prepPromise;
-    }
-
-    var sessionId = this._outboundSessionId;
+    var session = this._outboundSession;

    // need to make a brand new session?
-    if (!sessionId) {
-        this._prepPromise = this._prepareNewSession(room).
-            finally(function() {
-                self._prepPromise = null;
-            });
-        return this._prepPromise;
+    if (!session || session.needsRotation(self._sessionRotationPeriodMsgs,
+                                          self._sessionRotationPeriodMs)
+       ) {
+        this._outboundSession = session = this._prepareNewSession(room);
    }

-    if (this._sharePromise) {
+    if (session.sharePromise) {
        // key share already in progress
-        return this._sharePromise;
+        return session.sharePromise;
    }

-    // prep already done, but check for new devices
-    var shareMap = this._devicesPendingKeyShare;
-    this._devicesPendingKeyShare = {};
+    // no share in progress: check if we need to share with any devices
+    var prom = this._getDevicesInRoom(room).then(function(devicesInRoom) {
+        var shareMap = {};

-    // check each user is (still) a member of the room
-    for (var userId in shareMap) {
-        if (!shareMap.hasOwnProperty(userId)) {
-            continue;
+        for (var userId in devicesInRoom) {
+            if (!devicesInRoom.hasOwnProperty(userId)) {
+                continue;
+            }
+
+            var userDevices = devicesInRoom[userId];
+
+            for (var deviceId in userDevices) {
+                if (!userDevices.hasOwnProperty(deviceId)) {
+                    continue;
+                }
+
+                var deviceInfo = userDevices[deviceId];
+
+                if (deviceInfo.isBlocked()) {
+                    continue;
+                }
+
+                var key = deviceInfo.getIdentityKey();
+                if (key == self._olmDevice.deviceCurve25519Key) {
+                    // don't bother sending to ourself
+                    continue;
+                }
+
+                if (
+                    !session.sharedWithDevices[userId] ||
+                        session.sharedWithDevices[userId][deviceId] === undefined
+                ) {
+                    shareMap[userId] = shareMap[userId] || [];
+                    shareMap[userId].push(deviceInfo);
+                }
+            }
        }

-        // XXX what about rooms where invitees can see the content?
-        var member = room.getMember(userId);
-        if (member.membership !== "join") {
-            delete shareMap[userId];
-        }
-    }
-
-    this._sharePromise = this._shareKeyWithDevices(
-        sessionId, shareMap
-    ).finally(function() {
-        self._sharePromise = null;
+        return self._shareKeyWithDevices(
+            session, shareMap
+        );
+    }).finally(function() {
+        session.sharePromise = null;
    }).then(function() {
-        return sessionId;
+        return session;
    });

-    return this._sharePromise;
+    session.sharePromise = prom;
+    return prom;
 };

 /**
@@ -115,8 +190,7 @@ MegolmEncryption.prototype._ensureOutboundSession = function(room) {
 *
 * @param {module:models/room} room
 *
- * @return {module:client.Promise} Promise which resolves to the megolm
- *   sessionId when setup is complete.
+ * @return {module:crypto/algorithms/megolm.OutboundSessionInfo} session
 */
 MegolmEncryption.prototype._prepareNewSession = function(room) {
    var session_id = this._olmDevice.createOutboundGroupSession();
@@ -127,100 +201,53 @@ MegolmEncryption.prototype._prepareNewSession = function(room) {
        key.key, {ed25519: this._olmDevice.deviceEd25519Key}
    );

-    // we're going to share the key with all current members of the room,
-    // so we can reset this.
-    this._devicesPendingKeyShare = {};
-
-    var roomMembers = utils.map(room.getJoinedMembers(), function(u) {
-        return u.userId;
-    });
-
-    var shareMap = {};
-    for (var i = 0; i < roomMembers.length; i++) {
-        var userId = roomMembers[i];
-        shareMap[userId] = true;
-    }
-
-    var self = this;
-
-    // TODO: we need to give the user a chance to block any devices or users
-    // before we send them the keys; it's too late to download them here.
-    return this._crypto.downloadKeys(
-        roomMembers, false
-    ).then(function(res) {
-        return self._shareKeyWithDevices(session_id, shareMap);
-    }).then(function() {
-        if (self._discardNewSession) {
-            // we've had cause to reset the session_id since starting this process.
-            // we'll use the current session for any currently pending events, but
-            // don't save it as the current _outboundSessionId, so that new events
-            // will use a new session.
-            console.log("Session generation complete, but discarding");
-        } else {
-            self._outboundSessionId = session_id;
-        }
-        return session_id;
-    }).finally(function() {
-        self._discardNewSession = false;
-    });
+    return new OutboundSessionInfo(session_id);
 };

 /**
 * @private
 *
- * @param {string} session_id
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
 *
- * @param {Object<string, Object<string, boolean>|boolean>} shareMap
- *    Map from userid to either: true (meaning this is a new user in the room,
- *    so all of his devices need the keys); or a map from deviceid to true
- *    (meaning this user has one or more new devices, which need the keys).
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices
 *
 * @return {module:client.Promise} Promise which resolves once the key sharing
 *     message has been sent.
 */
-MegolmEncryption.prototype._shareKeyWithDevices = function(session_id, shareMap) {
+MegolmEncryption.prototype._shareKeyWithDevices = function(session, devicesByUser) {
    var self = this;

-    var key = this._olmDevice.getOutboundGroupSessionKey(session_id);
+    var key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
    var payload = {
        type: "m.room_key",
        content: {
            algorithm: olmlib.MEGOLM_ALGORITHM,
            room_id: this._roomId,
-            session_id: session_id,
+            session_id: session.sessionId,
            session_key: key.key,
            chain_index: key.chain_index,
        }
    };

-    // we downloaded the user's device list when they joined the room, or when
-    // the new device announced itself, so there is no need to do so now.
+    var contentMap = {};

-    return self._crypto.ensureOlmSessionsForUsers(
-        utils.keys(shareMap)
+    return olmlib.ensureOlmSessionsForDevices(
+        this._olmDevice, this._baseApis, devicesByUser
    ).then(function(devicemap) {
-        var contentMap = {};
        var haveTargets = false;

-        for (var userId in devicemap) {
-            if (!devicemap.hasOwnProperty(userId)) {
+        for (var userId in devicesByUser) {
+            if (!devicesByUser.hasOwnProperty(userId)) {
                continue;
            }

-            var devicesToShareWith = shareMap[userId];
+            var devicesToShareWith = devicesByUser[userId];
            var sessionResults = devicemap[userId];

-            for (var deviceId in sessionResults) {
-                if (!sessionResults.hasOwnProperty(deviceId)) {
-                    continue;
-                }
-
-                if (devicesToShareWith === true) {
-                    // all devices
-                } else if (!devicesToShareWith[deviceId]) {
-                    // not a new device
-                    continue;
-                }
+            for (var i = 0; i < devicesToShareWith.length; i++) {
+                var deviceInfo = devicesToShareWith[i];
+                var deviceId = deviceInfo.deviceId;

                var sessionResult = sessionResults[deviceId];
                if (!sessionResult.sessionId) {
@@ -242,19 +269,27 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session_id, shareMap)
                    "sharing keys with device " + userId + ":" + deviceId
                );

-                var deviceInfo = sessionResult.device;
+                var encryptedContent = {
+                    algorithm: olmlib.OLM_ALGORITHM,
+                    sender_key: self._olmDevice.deviceCurve25519Key,
+                    ciphertext: {},
+                };
+
+                olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    self._userId,
+                    self._deviceId,
+                    self._olmDevice,
+                    userId,
+                    deviceInfo,
+                    payload
+                );

                if (!contentMap[userId]) {
                    contentMap[userId] = {};
                }

-                contentMap[userId][deviceId] =
-                    olmlib.encryptMessageForDevices(
-                        self._deviceId,
-                        self._olmDevice,
-                        [deviceInfo.getIdentityKey()],
-                        payload
-                    );
+                contentMap[userId][deviceId] = encryptedContent;
                haveTargets = true;
            }
        }
@@ -265,6 +300,27 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session_id, shareMap)

        // TODO: retries
        return self._baseApis.sendToDevice("m.room.encrypted", contentMap);
+    }).then(function() {
+        // Add the devices we have shared with to session.sharedWithDevices.
+        //
+        // we deliberately iterate over devicesByUser (ie, the devices we
+        // attempted to share with) rather than the contentMap (those we did
+        // share with), because we don't want to try to claim a one-time-key
+        // for dead devices on every message.
+        for (var userId in devicesByUser) {
+            if (!devicesByUser.hasOwnProperty(userId)) {
+                continue;
+            }
+            if (!session.sharedWithDevices[userId]) {
+                session.sharedWithDevices[userId] = {};
+            }
+            var devicesToShareWith = devicesByUser[userId];
+            for (var i = 0; i < devicesToShareWith.length; i++) {
+                var deviceInfo = devicesToShareWith[i];
+                session.sharedWithDevices[userId][deviceInfo.deviceId] =
+                    key.chain_index;
+            }
+        }
    });
 };

@@ -279,7 +335,7 @@ MegolmEncryption.prototype._shareKeyWithDevices = function(session_id, shareMap)
 */
 MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
    var self = this;
-    return this._ensureOutboundSession(room).then(function(session_id) {
+    return this._ensureOutboundSession(room).then(function(session) {
        var payloadJson = {
            room_id: self._roomId,
            type: eventType,
@@ -287,19 +343,20 @@ MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
        };

        var ciphertext = self._olmDevice.encryptGroupMessage(
-            session_id, JSON.stringify(payloadJson)
+            session.sessionId, JSON.stringify(payloadJson)
        );

        var encryptedContent = {
            algorithm: olmlib.MEGOLM_ALGORITHM,
            sender_key: self._olmDevice.deviceCurve25519Key,
            ciphertext: ciphertext,
-            session_id: session_id,
+            session_id: session.sessionId,
             // Include our device ID so that recipients can send us a
             // m.new_device message if they don't have our session key.
            device_id: self._deviceId,
        };

+        session.useCount++;
        return encryptedContent;
    });
 };
@@ -314,71 +371,40 @@ MegolmEncryption.prototype.encryptMessage = function(room, eventType, content) {
 MegolmEncryption.prototype.onRoomMembership = function(event, member, oldMembership) {
    var newMembership = member.membership;

-    if (newMembership === 'join') {
-        this._onNewRoomMember(member.userId);
-        return;
-    }
-
-    if (newMembership === 'invite' && oldMembership !== 'join') {
-        // we don't (yet) share keys with invited members, so nothing to do yet
+    if (newMembership === 'join' || newMembership === 'invite') {
        return;
    }

    // otherwise we assume the user is leaving, and start a new outbound session.
-    if (this._outboundSessionId) {
-        console.log("Discarding outbound megolm session due to change in " +
-                    "membership of " + member.userId + " (" + oldMembership +
-                    "->" + newMembership + ")");
-        this._outboundSessionId = null;
-    }
+    console.log("Discarding outbound megolm session due to change in " +
+                "membership of " + member.userId + " (" + oldMembership +
+                "->" + newMembership + ")");

-    if (this._prepPromise) {
-        console.log("Discarding as-yet-incomplete megolm session due to " +
-                    "change in membership of " + member.userId + " (" +
-                    oldMembership + "->" + newMembership + ")");
-        this._discardNewSession = true;
-    }
+    // this ensures that we will start a new session on the next message.
+    this._outboundSession = null;
 };

 /**
- * handle a new user joining a room
+ * Get the list of devices for all users in the room
 *
- * @param {string} userId   new member
- */
-MegolmEncryption.prototype._onNewRoomMember = function(userId) {
-    // make sure we have a list of this user's devices. We are happy to use a
-    // cached version here: we assume that if we already have a list of the
-    // user's devices, then we already share an e2e room with them, which means
-    // that they will have announced any new devices via an m.new_device.
-    this._crypto.downloadKeys([userId], false).done();
-
-    // also flag this user up for needing a keyshare.
-    this._devicesPendingKeyShare[userId] = true;
-};
-
-
-/**
- * @inheritdoc
+ * @param {module:models/room} room
 *
- * @param {string} userId    owner of the device
- * @param {string} deviceId  deviceId of the device
+ * @return {module:client.Promise} Promise which resolves to a map
+ *     from userId to deviceId to deviceInfo
 */
-MegolmEncryption.prototype.onNewDevice = function(userId, deviceId) {
-    var d = this._devicesPendingKeyShare[userId];
+MegolmEncryption.prototype._getDevicesInRoom = function(room) {
+    // XXX what about rooms where invitees can see the content?
+    var roomMembers = utils.map(room.getJoinedMembers(), function(u) {
+        return u.userId;
+    });

-    if (d === true) {
-        // we already want to share keys with all devices for this user
-        return;
-    }
-
-    if (!d) {
-        this._devicesPendingKeyShare[userId] = d = {};
-    }
-
-    d[deviceId] = true;
+    // We are happy to use a cached version here: we assume that if we already
+    // have a list of the user's devices, then we already share an e2e room
+    // with them, which means that they will have announced any new devices via
+    // an m.new_device.
+    return this._crypto.downloadKeys(roomMembers, false);
 };

-
 /**
 * Megolm decryption implementation
 *
@@ -390,13 +416,17 @@ MegolmEncryption.prototype.onNewDevice = function(userId, deviceId) {
 */
 function MegolmDecryption(params) {
    base.DecryptionAlgorithm.call(this, params);
+
+    // events which we couldn't decrypt due to unknown sessions / indexes: map from
+    // senderKey|sessionId to list of MatrixEvents
+    this._pendingEvents = {};
 }
 utils.inherits(MegolmDecryption, base.DecryptionAlgorithm);

 /**
 * @inheritdoc
 *
- * @param {object} event raw event
+ * @param {MatrixEvent} event
 *
 * @return {null} The event referred to an unknown megolm session
 * @return {module:crypto.DecryptionResult} decryption result
@@ -405,7 +435,7 @@ utils.inherits(MegolmDecryption, base.DecryptionAlgorithm);
 *   problem decrypting the event
 */
 MegolmDecryption.prototype.decryptEvent = function(event) {
-    var content = event.content;
+    var content = event.getWireContent();

    if (!content.sender_key || !content.session_id ||
        !content.ciphertext
@@ -413,21 +443,56 @@ MegolmDecryption.prototype.decryptEvent = function(event) {
        throw new base.DecryptionError("Missing fields in input");
    }

+    var res;
    try {
-        var res = this._olmDevice.decryptGroupMessage(
-            event.room_id, content.sender_key, content.session_id, content.ciphertext
+        res = this._olmDevice.decryptGroupMessage(
+            event.getRoomId(), content.sender_key, content.session_id, content.ciphertext
        );
-        if (res === null) {
-            return null;
-        }
-        return {
-            payload: JSON.parse(res.result),
-            keysClaimed: res.keysClaimed,
-            keysProved: res.keysProved,
-        };
    } catch (e) {
+        if (e.message === 'OLM.UNKNOWN_MESSAGE_INDEX') {
+            this._addEventToPendingList(event);
+        }
        throw new base.DecryptionError(e);
    }
+
+    if (res === null) {
+        // We've got a message for a session we don't have.
+        this._addEventToPendingList(event);
+        throw new base.DecryptionError(
+            "The sender's device has not sent us the keys for this message."
+        );
+    }
+
+    var payload = JSON.parse(res.result);
+
+    // belt-and-braces check that the room id matches that indicated by the HS
+    // (this is somewhat redundant, since the megolm session is scoped to the
+    // room, so neither the sender nor a MITM can lie about the room_id).
+    if (payload.room_id !== event.getRoomId()) {
+        throw new base.DecryptionError(
+            "Message intended for room " + payload.room_id
+        );
+    }
+
+    event.setClearData(payload, res.keysProved, res.keysClaimed);
+};
+
+
+/**
+ * Add an event to the list of those we couldn't decrypt the first time we
+ * saw them.
+ *
+ * @private
+ *
+ * @param {module:models/event.MatrixEvent} event
+ */
+MegolmDecryption.prototype._addEventToPendingList = function(event) {
+    var content = event.getWireContent();
+    var k = content.sender_key + "|" + content.session_id;
+    if (!this._pendingEvents[k]) {
+        this._pendingEvents[k] = [];
+    }
+    this._pendingEvents[k].push(event);
 };

 /**
@@ -451,6 +516,22 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        content.room_id, event.getSenderKey(), content.session_id,
        content.session_key, event.getKeysClaimed()
    );
+
+    var k = event.getSenderKey() + "|" + content.session_id;
+    var pending = this._pendingEvents[k];
+    if (pending) {
+        // have another go at decrypting events sent with this session.
+        delete this._pendingEvents[k];
+
+        for (var i = 0; i < pending.length; i++) {
+            try {
+                this.decryptEvent(pending[i]);
+                console.log("successful re-decryption of", pending[i]);
+            } catch (e) {
+                console.log("Still can't decrypt", pending[i], e.stack || e);
+            }
+        }
+    }
 };

 base.registerAlgorithm(
@@ -95,32 +95,43 @@ OlmEncryption.prototype.encryptMessage = function(room, eventType, content) {

    var self = this;
    return this._ensureSession(users).then(function() {
-        var participantKeys = [];
+        var payloadFields = {
+            room_id: room.roomId,
+            type: eventType,
+            content: content,
+        };
+
+        var encryptedContent = {
+            algorithm: olmlib.OLM_ALGORITHM,
+            sender_key: self._olmDevice.deviceCurve25519Key,
+            ciphertext: {},
+        };
+
        for (var i = 0; i < users.length; ++i) {
            var userId = users[i];
            var devices = self._crypto.getStoredDevicesForUser(userId);
+
            for (var j = 0; j < devices.length; ++j) {
                var deviceInfo = devices[j];
                var key = deviceInfo.getIdentityKey();
                if (key == self._olmDevice.deviceCurve25519Key) {
-                    // don't bother setting up session to ourself
+                    // don't bother sending to ourself
                    continue;
                }
                if (deviceInfo.verified == DeviceVerification.BLOCKED) {
                    // don't bother setting up sessions with blocked users
                    continue;
                }
-                participantKeys.push(key);
+
+                olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    self._userId, self._deviceId, self._olmDevice,
+                    userId, deviceInfo, payloadFields
+                );
            }
        }

-        return olmlib.encryptMessageForDevices(
-            self._deviceId, self._olmDevice, participantKeys, {
-                room_id: room.roomId,
-                type: eventType,
-                content: content,
-            }
-        );
+        return encryptedContent;
    });
 };

@@ -140,15 +151,13 @@ utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
 /**
 * @inheritdoc
 *
- * @param {object} event raw event
- *
- * @return {module:crypto.DecryptionResult} decryption result
+ * @param {MatrixEvent} event
 *
 * @throws {module:crypto/algorithms/base.DecryptionError} if there is a
 *   problem decrypting the event
 */
 OlmDecryption.prototype.decryptEvent = function(event) {
-    var content = event.content;
+    var content = event.getWireContent();
    var deviceKey = content.sender_key;
    var ciphertext = content.ciphertext;

@@ -167,22 +176,61 @@ OlmDecryption.prototype.decryptEvent = function(event) {
    } catch (e) {
        console.warn(
            "Failed to decrypt Olm event (id=" +
-                event.event_id + ") from " + deviceKey +
+                event.getId() + ") from " + deviceKey +
                ": " + e.message
        );
        throw new base.DecryptionError("Bad Encrypted Message");
    }

-
-    // TODO: Check the sender user id matches the sender key.
-    // TODO: check the room_id and fingerprint
    var payload = JSON.parse(payloadString);
-    return {
-        payload: payload,
-        sessionExists: true,
-        keysProved: {curve25519: deviceKey},
-        keysClaimed: payload.keys || {}
-    };
+
+    // check that we were the intended recipient, to avoid unknown-key attack
+    // https://github.com/vector-im/vector-web/issues/2483
+    if (payload.recipient != this._userId) {
+        console.warn(
+            "Event " + event.getId() + ": Intended recipient " +
+            payload.recipient + " does not match our id " + this._userId
+        );
+        throw new base.DecryptionError(
+            "Message was intented for " + payload.recipient
+        );
+    }
+
+    if (payload.recipient_keys.ed25519 !=
+               this._olmDevice.deviceEd25519Key) {
+        console.warn(
+            "Event " + event.getId() + ": Intended recipient ed25519 key " +
+            payload.recipient_keys.ed25519 + " did not match ours"
+        );
+        throw new base.DecryptionError("Message not intended for this device");
+    }
+
+    // check that the original sender matches what the homeserver told us, to
+    // avoid people masquerading as others.
+    // (this check is also provided via the sender's embedded ed25519 key,
+    // which is checked elsewhere).
+    if (payload.sender != event.getSender()) {
+        console.warn(
+            "Event " + event.getId() + ": original sender " + payload.sender +
+            " does not match reported sender " + event.getSender()
+        );
+        throw new base.DecryptionError(
+            "Message forwarded from " + payload.sender
+        );
+    }
+
+    // Olm events intended for a room have a room_id.
+    if (payload.room_id !== event.getRoomId()) {
+        console.warn(
+            "Event " + event.getId() + ": original room " + payload.room_id +
+            " does not match reported room " + event.room_id
+        );
+        throw new base.DecryptionError(
+            "Message intended for room " + payload.room_id
+        );
+    }
+
+    event.setClearData(payload, {curve25519: deviceKey}, payload.keys || {});
 };


@@ -54,10 +54,17 @@ function Crypto(baseApis, eventEmitter, sessionStore, userId, deviceId) {
    this._userId = userId;
    this._deviceId = deviceId;

+    this._initialSyncCompleted = false;
+    // userId -> deviceId -> true
+    this._pendingNewDevices = {};
+
    this._olmDevice = new OlmDevice(sessionStore);

    // EncryptionAlgorithm instance for each room
-    this._roomAlgorithms = {};
+    this._roomEncryptors = {};
+
+    // map from algorithm to DecryptionAlgorithm instance, for each room
+    this._roomDecryptors = {};

    this._supportedAlgorithms = utils.keys(
        algorithms.DECRYPTION_CLASSES
@@ -174,7 +181,7 @@ Crypto.prototype.uploadKeys = function(maxKeys) {
        // these factors.

        // We first find how many keys the server has for us.
-        var keyCount = res.one_time_key_counts.curve25519 || 0;
+        var keyCount = res.one_time_key_counts.signed_curve25519 || 0;
        // We then check how many keys we can store in the Account object.
        var maxOneTimeKeys = self._olmDevice.maxNumberOfOneTimeKeys();
        // Try to keep at most half that number on the server. This leaves the
@@ -217,11 +224,7 @@ function _uploadDeviceKeys(crypto) {
        keys: crypto._deviceKeys,
        user_id: userId,
    };
-
-    var sig = crypto._olmDevice.sign(anotherjson.stringify(deviceKeys));
-    deviceKeys.signatures = {};
-    deviceKeys.signatures[userId] = {};
-    deviceKeys.signatures[userId]["ed25519:" + deviceId] = sig;
+    crypto._signObject(deviceKeys);

    return crypto._baseApis.uploadKeysRequest({
        device_keys: deviceKeys,
@@ -239,9 +242,14 @@ function _uploadOneTimeKeys(crypto) {

    for (var keyId in oneTimeKeys.curve25519) {
        if (oneTimeKeys.curve25519.hasOwnProperty(keyId)) {
-            oneTimeJson["curve25519:" + keyId] = oneTimeKeys.curve25519[keyId];
+            var k = {
+                key: oneTimeKeys.curve25519[keyId],
+            };
+            crypto._signObject(k);
+            oneTimeJson["signed_curve25519:" + keyId] = k;
        }
    }
+
    return crypto._baseApis.uploadKeysRequest({
        one_time_keys: oneTimeJson
    }, {
@@ -268,22 +276,26 @@ Crypto.prototype.downloadKeys = function(userIds, forceDownload) {

    // map from userid -> deviceid -> DeviceInfo
    var stored = {};
+    function storeDev(userId, dev) {
+        stored[userId][dev.deviceId] = dev;
+    }

    // list of userids we need to download keys for
    var downloadUsers = [];

-    for (var i = 0; i < userIds.length; ++i) {
-        var userId = userIds[i];
-        stored[userId] = {};
+    if (forceDownload) {
+        downloadUsers = userIds;
+    } else {
+        for (var i = 0; i < userIds.length; ++i) {
+            var userId = userIds[i];
+            var devices = this.getStoredDevicesForUser(userId);

-        var devices = this.getStoredDevicesForUser(userId);
-        for (var j = 0; j < devices.length; ++j) {
-            var dev = devices[j];
-            stored[userId][dev.deviceId] = dev;
-        }
-
-        if (devices.length === 0 || forceDownload) {
-            downloadUsers.push(userId);
+            if (!devices) {
+                downloadUsers.push(userId);
+            } else {
+                stored[userId] = {};
+                devices.map(storeDev.bind(null, userId));
+            }
        }
    }

@@ -291,40 +303,98 @@ Crypto.prototype.downloadKeys = function(userIds, forceDownload) {
        return q(stored);
    }

-    return this._baseApis.downloadKeysForUsers(
+    var r = this._doKeyDownloadForUsers(downloadUsers);
+    var promises = [];
+    downloadUsers.map(function(u) {
+        promises.push(r[u].catch(function(e) {
+            console.warn('Error downloading keys for user ' + u + ':', e);
+        }).then(function() {
+            stored[u] = {};
+            var devices = self.getStoredDevicesForUser(u) || [];
+            devices.map(storeDev.bind(null, u));
+        }));
+    });
+
+    return q.all(promises).then(function() {
+        return stored;
+    });
+};
+
+/**
+ * @param {string[]} downloadUsers list of userIds
+ *
+ * @return {Object a map from userId to a promise for a result for that user
+ */
+Crypto.prototype._doKeyDownloadForUsers = function(downloadUsers) {
+    var self = this;
+
+    console.log('Starting key download for ' + downloadUsers);
+
+    var deferMap = {};
+    var promiseMap = {};
+
+    downloadUsers.map(function(u) {
+        deferMap[u] = q.defer();
+        promiseMap[u] = deferMap[u].promise;
+    });
+
+    this._baseApis.downloadKeysForUsers(
        downloadUsers
-    ).then(function(res) {
-        for (var userId in res.device_keys) {
-            if (!stored.hasOwnProperty(userId)) {
-                // spurious result
+    ).done(function(res) {
+        var dk = res.device_keys || {};
+
+        for (var i = 0; i < downloadUsers.length; ++i) {
+            var userId = downloadUsers[i];
+            var deviceId;
+
+            console.log('got keys for ' + userId + ':', dk[userId]);
+
+            if (!dk[userId]) {
+                // no result for this user
+                var err = 'Unknown';
+                // TODO: do something with res.failures
+                deferMap[userId].reject(err);
                continue;
            }

            // map from deviceid -> deviceinfo for this user
-            var userStore = stored[userId];
-            var updated = _updateStoredDeviceKeysForUser(
-                self._olmDevice, userId, userStore, res.device_keys[userId]
-            );
-
-            if (!updated) {
-                continue;
+            var userStore = {};
+            var devs = self._sessionStore.getEndToEndDevicesForUser(userId);
+            if (devs) {
+                for (deviceId in devs) {
+                    if (devs.hasOwnProperty(deviceId)) {
+                        var d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                        userStore[deviceId] = d;
+                    }
+                }
            }

+            _updateStoredDeviceKeysForUser(
+                self._olmDevice, userId, userStore, dk[userId]
+            );
+
            // update the session store
            var storage = {};
-            for (var deviceId in userStore) {
+            for (deviceId in userStore) {
                if (!userStore.hasOwnProperty(deviceId)) {
                    continue;
                }

                storage[deviceId] = userStore[deviceId].toStorage();
-                self._sessionStore.storeEndToEndDevicesForUser(
-                    userId, storage
-                );
            }
+            self._sessionStore.storeEndToEndDevicesForUser(
+                userId, storage
+            );
+
+            deferMap[userId].resolve();
        }
-        return stored;
+    }, function(err) {
+        downloadUsers.map(function(u) {
+            deferMap[u].reject(err);
+        });
    });
+
+    return promiseMap;
 };

 function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
@@ -350,9 +420,22 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
            continue;
        }

-        if (_storeDeviceKeys(
-            _olmDevice, userId, deviceId, userStore, userResult[deviceId]
-        )) {
+        var deviceResult = userResult[deviceId];
+
+        // check that the user_id and device_id in the response object are
+        // correct
+        if (deviceResult.user_id !== userId) {
+            console.warn("Mismatched user_id " + deviceResult.user_id +
+                         " in keys from " + userId + ":" + deviceId);
+            continue;
+        }
+        if (deviceResult.device_id !== deviceId) {
+            console.warn("Mismatched device_id " + deviceResult.device_id +
+                         " in keys from " + userId + ":" + deviceId);
+            continue;
+        }
+
+        if (_storeDeviceKeys(_olmDevice, userStore, deviceResult)) {
            updated = true;
        }
    }
@@ -365,12 +448,15 @@ function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
 *
 * returns true if a change was made, else false
 */
-function _storeDeviceKeys(_olmDevice, userId, deviceId, userStore, deviceResult) {
+function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
    if (!deviceResult.keys) {
        // no keys?
        return false;
    }

+    var deviceId = deviceResult.device_id;
+    var userId = deviceResult.user_id;
+
    var signKeyId = "ed25519:" + deviceId;
    var signKey = deviceResult.keys[signKeyId];
    if (!signKey) {
@@ -380,23 +466,9 @@ function _storeDeviceKeys(_olmDevice, userId, deviceId, userStore, deviceResult)
    }

    var unsigned = deviceResult.unsigned || {};
-    var signatures = deviceResult.signatures || {};
-    var userSigs = signatures[userId] || {};
-    var signature = userSigs[signKeyId];
-    if (!signature) {
-        console.log("Device " + userId + ":" + deviceId +
-                    " is not signed");
-        return false;
-    }
-
-    // prepare the canonical json: remove 'unsigned' and signatures, and
-    // stringify with anotherjson
-    delete deviceResult.unsigned;
-    delete deviceResult.signatures;
-    var json = anotherjson.stringify(deviceResult);

    try {
-        _olmDevice.verifySignature(signKey, json, signature);
+        olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
    } catch (e) {
        console.log("Unable to verify signature on device " +
                    userId + ":" + deviceId + ":", e);
@@ -434,12 +506,13 @@ function _storeDeviceKeys(_olmDevice, userId, deviceId, userStore, deviceResult)
 *
 * @param {string} userId the user to list keys for.
 *
- * @return {module:crypto/deviceinfo[]} list of devices
+ * @return {module:crypto/deviceinfo[]?} list of devices, or null if we haven't
+ * managed to get a list of devices for this user yet.
 */
 Crypto.prototype.getStoredDevicesForUser = function(userId) {
    var devs = this._sessionStore.getEndToEndDevicesForUser(userId);
    if (!devs) {
-        return [];
+        return null;
    }
    var res = [];
    for (var deviceId in devs) {
@@ -450,6 +523,22 @@ Crypto.prototype.getStoredDevicesForUser = function(userId) {
    return res;
 };

+/**
+ * Get the stored keys for a single device
+ *
+ * @param {string} userId
+ * @param {string} deviceId
+ *
+ * @return {module:crypto/deviceinfo?} list of devices, or undefined
+ * if we don't know about this device
+ */
+Crypto.prototype.getStoredDevice = function(userId, deviceId) {
+    var devs = this._sessionStore.getEndToEndDevicesForUser(userId);
+    if (!devs || !devs[deviceId]) {
+        return undefined;
+    }
+    return DeviceInfo.fromStorage(devs[deviceId], deviceId);
+};

 /**
 * List the stored device keys for a user id
@@ -462,7 +551,7 @@ Crypto.prototype.getStoredDevicesForUser = function(userId) {
 *    "key", and "display_name" parameters.
 */
 Crypto.prototype.listDeviceKeys = function(userId) {
-    var devices = this.getStoredDevicesForUser(userId);
+    var devices = this.getStoredDevicesForUser(userId) || [];

    var result = [];

@@ -594,7 +683,7 @@ Crypto.prototype.setDeviceVerification = function(userId, deviceId, verified, bl
 * @return {Object.<string, {deviceIdKey: string, sessions: object[]}>}
 */
 Crypto.prototype.getOlmSessionsForUser = function(userId) {
-    var devices = this.getStoredDevicesForUser(userId);
+    var devices = this.getStoredDevicesForUser(userId) || [];
    var result = {};
    for (var j = 0; j < devices.length; ++j) {
        var device = devices[j];
@@ -694,13 +783,15 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
    this._sessionStore.storeEndToEndRoom(roomId, config);

    var alg = new AlgClass({
+        userId: this._userId,
        deviceId: this._deviceId,
        crypto: this,
        olmDevice: this._olmDevice,
        baseApis: this._baseApis,
        roomId: roomId,
+        config: config,
    });
-    this._roomAlgorithms[roomId] = alg;
+    this._roomEncryptors[roomId] = alg;
 };


@@ -712,7 +803,8 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
 */

 /**
- * Try to make sure we have established olm sessions for the given users.
+ * Try to make sure we have established olm sessions for all known devices for
+ * the given users.
 *
 * @param {string[]} users list of user ids
 *
@@ -721,19 +813,15 @@ Crypto.prototype.setRoomEncryption = function(roomId, config) {
 *    {@link module:crypto~OlmSessionResult}
 */
 Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
-    var devicesWithoutSession = [
-        // [userId, deviceId, deviceInfo], ...
-    ];
-    var result = {};
+    var devicesByUser = {};

    for (var i = 0; i < users.length; ++i) {
        var userId = users[i];
-        result[userId] = {};
+        devicesByUser[userId] = [];

-        var devices = this.getStoredDevicesForUser(userId);
+        var devices = this.getStoredDevicesForUser(userId) || [];
        for (var j = 0; j < devices.length; ++j) {
            var deviceInfo = devices[j];
-            var deviceId = deviceInfo.deviceId;

            var key = deviceInfo.getIdentityKey();
            if (key == this._olmDevice.deviceCurve25519Key) {
@@ -745,60 +833,13 @@ Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
                continue;
            }

-            var sessionId = this._olmDevice.getSessionIdForDevice(key);
-            if (sessionId === null) {
-                devicesWithoutSession.push([userId, deviceId, deviceInfo]);
-            }
-            result[userId][deviceId] = {
-                device: deviceInfo,
-                sessionId: sessionId,
-            };
+            devicesByUser[userId].push(deviceInfo);
        }
    }

-    if (devicesWithoutSession.length === 0) {
-        return q(result);
-    }
-
-    // TODO: this has a race condition - if we try to send another message
-    // while we are claiming a key, we will end up claiming two and setting up
-    // two sessions.
-    //
-    // That should eventually resolve itself, but it's poor form.
-
-    var self = this;
-    return this._baseApis.claimOneTimeKeys(
-        devicesWithoutSession
-    ).then(function(res) {
-        for (var i = 0; i < devicesWithoutSession.length; ++i) {
-            var device = devicesWithoutSession[i];
-            var userId = device[0];
-            var deviceId = device[1];
-            var deviceInfo = device[2];
-
-            var userRes = res.one_time_keys[userId] || {};
-            var deviceRes = userRes[deviceId];
-            var oneTimeKey = null;
-            for (var keyId in deviceRes) {
-                if (keyId.indexOf("curve25519:") === 0) {
-                    oneTimeKey = deviceRes[keyId];
-                }
-            }
-            if (oneTimeKey) {
-                var sid = self._olmDevice.createOutboundSession(
-                    deviceInfo.getIdentityKey(), oneTimeKey
-                );
-                console.log("Started new sessionid " + sid +
-                            " for device " + userId + ":" + deviceId);
-
-                result[userId][deviceId].sessionId = sid;
-            } else {
-                console.warn("No one-time keys for device " +
-                             userId + ":" + deviceId);
-            }
-        }
-        return result;
-    });
+    return olmlib.ensureOlmSessionsForDevices(
+        this._olmDevice, this._baseApis, devicesByUser
+    );
 };

 /**
@@ -807,10 +848,9 @@ Crypto.prototype.ensureOlmSessionsForUsers = function(users) {
 * @return {bool} whether encryption is enabled.
 */
 Crypto.prototype.isRoomEncrypted = function(roomId) {
-    return Boolean(this._roomAlgorithms[roomId]);
+    return Boolean(this._roomEncryptors[roomId]);
 };

-
 /**
 * Encrypt an event according to the configuration of the room, if necessary.
 *
@@ -830,18 +870,13 @@ Crypto.prototype.encryptEventIfNeeded = function(event, room) {
        return null;
    }

-    if (event.getType() !== "m.room.message") {
-        // we only encrypt m.room.message
-        return null;
-    }
-
    if (!room) {
        throw new Error("Cannot send encrypted messages in unknown rooms");
    }

    var roomId = event.getRoomId();

-    var alg = this._roomAlgorithms[roomId];
+    var alg = this._roomEncryptors[roomId];
    if (!alg) {
        // not encrypting messages in this room

@@ -871,65 +906,17 @@ Crypto.prototype.encryptEventIfNeeded = function(event, room) {
    });
 };

-/**
- * @typedef {Object} module:crypto.DecryptionResult
- *
- * @property {Object} payload decrypted payload (with properties 'type',
- *     'content').
- *
- * @property {Object<string, string>} keysClaimed keys that the sender of the
- *     event claims ownership of: map from key type to base64-encoded key
- *
- * @property {Object<string, string>} keysProved keys that the sender of the
- *     event is known to have ownership of: map from key type to base64-encoded
- *     key
- */
-
 /**
 * Decrypt a received event
 *
- * @param {object} event raw event
- *
- * @return {module:crypto.DecryptionResult} decryption result
+ * @param {MatrixEvent} event
 *
 * @raises {algorithms.DecryptionError} if there is a problem decrypting the event
 */
 Crypto.prototype.decryptEvent = function(event) {
-    var content = event.content;
-    var AlgClass = algorithms.DECRYPTION_CLASSES[content.algorithm];
-    if (!AlgClass) {
-        throw new algorithms.DecryptionError("Unable to decrypt " + content.algorithm);
-    }
-    var alg = new AlgClass({
-        olmDevice: this._olmDevice,
-    });
-    var r = alg.decryptEvent(event);
-
-    if (r !== null) {
-        return r;
-    } else {
-        // We've got a message for a session we don't have.  Maybe the sender
-        // forgot to tell us about the session.  Remind the sender that we
-        // exist so that they might tell us about the session on their next
-        // send.
-        //
-        // (Alternatively, it might be that we are just looking at
-        // scrollback... at least we rate-limit the m.new_device events :/)
-        //
-        // XXX: this is a band-aid which masks symptoms of other bugs. It would
-        // be nice to get rid of it.
-        if (event.room_id !== undefined && event.sender !== undefined) {
-            var device_id = event.content.device_id;
-            if (device_id === undefined) {
-                // if the sending device didn't tell us its device_id, fall
-                // back to all devices.
-                device_id = null;
-            }
-            this._sendPingToDevice(event.sender, device_id, event.room_id);
-        }
-
-        throw new algorithms.DecryptionError("Unknown inbound session id");
-    }
+    var content = event.getWireContent();
+    var alg = this._getRoomDecryptor(event.getRoomId(), content.algorithm);
+    alg.decryptEvent(event);
 };

 /**
@@ -1011,6 +998,11 @@ Crypto.prototype._onCryptoEvent = function(event) {
 * @param {module:models/room[]} rooms list of rooms the client knows about
 */
 Crypto.prototype._onInitialSyncCompleted = function(rooms) {
+    this._initialSyncCompleted = true;
+
+    // catch up on any m.new_device events which arrived during the initial sync.
+    this._flushNewDeviceRequests();
+
    if (this._sessionStore.getDeviceAnnounced()) {
        return;
    }
@@ -1023,7 +1015,7 @@ Crypto.prototype._onInitialSyncCompleted = function(rooms) {
        var room = rooms[i];

        // check for rooms with encryption enabled
-        var alg = this._roomAlgorithms[room.roomId];
+        var alg = this._roomEncryptors[room.roomId];
        if (!alg) {
            continue;
        }
@@ -1077,15 +1069,13 @@ Crypto.prototype._onInitialSyncCompleted = function(rooms) {
 */
 Crypto.prototype._onRoomKeyEvent = function(event) {
    var content = event.getContent();
-    var AlgClass = algorithms.DECRYPTION_CLASSES[content.algorithm];
-    if (!AlgClass) {
-        throw new algorithms.DecryptionError(
-            "Unable to handle keys for " + content.algorithm
-        );
+
+    if (!content.room_id || !content.algorithm) {
+        console.error("key event is missing fields");
+        return;
    }
-    var alg = new AlgClass({
-        olmDevice: this._olmDevice,
-    });
+
+    var alg = this._getRoomDecryptor(content.room_id, content.algorithm);
    alg.onRoomKeyEvent(event);
 };

@@ -1109,7 +1099,7 @@ Crypto.prototype._onRoomMembership = function(event, member, oldMembership) {

    var roomId = member.roomId;

-    var alg = this._roomAlgorithms[roomId];
+    var alg = this._roomEncryptors[roomId];
    if (!alg) {
        // not encrypting in this room
        return;
@@ -1139,26 +1129,126 @@ Crypto.prototype._onNewDeviceEvent = function(event) {
    console.log("m.new_device event from " + userId + ":" + deviceId +
                " for rooms " + rooms);

+    if (this.getStoredDevice(userId, deviceId)) {
+        console.log("Known device; ignoring");
+        return;
+    }
+
+    this._pendingNewDevices[userId] = this._pendingNewDevices[userId] || {};
+    this._pendingNewDevices[userId][deviceId] = true;
+
+    // we delay handling these until the intialsync has completed, so that we
+    // can do all of them together.
+    if (this._initialSyncCompleted) {
+        this._flushNewDeviceRequests();
+    }
+};
+
+/**
+ * Start device queries for any users who sent us an m.new_device recently
+ */
+Crypto.prototype._flushNewDeviceRequests = function() {
    var self = this;
-    this.downloadKeys(
-        [userId], true
-    ).then(function() {
-        for (var i = 0; i < rooms.length; i++) {
-            var roomId = rooms[i];
-            var alg = self._roomAlgorithms[roomId];
-            if (!alg) {
-                // not encrypting in this room
-                continue;
-            }
-            alg.onNewDevice(userId, deviceId);
+
+    var pending = this._pendingNewDevices;
+    var users = utils.keys(pending).filter(function(u) {
+        return utils.keys(pending[u]).length > 0;
+    });
+
+    if (users.length === 0) {
+        return;
+    }
+
+    var r = this._doKeyDownloadForUsers(users);
+
+    // we've kicked off requests to these users: remove their
+    // pending flag for now.
+    this._pendingNewDevices = {};
+
+    users.map(function(u) {
+        r[u] = r[u].catch(function(e) {
+            console.error(
+                'Error updating device keys for user ' + u + ':', e
+            );
+
+            // reinstate the pending flags on any users which failed; this will
+            // mean that we will do another download in the future, but won't
+            // tight-loop.
+            //
+            self._pendingNewDevices[u] = self._pendingNewDevices[u] || {};
+            utils.update(self._pendingNewDevices[u], pending[u]);
+        });
+    });
+
+    q.all(utils.values(r)).done();
+};
+
+/**
+ * Get a decryptor for a given room and algorithm.
+ *
+ * If we already have a decryptor for the given room and algorithm, return
+ * it. Otherwise try to instantiate it.
+ *
+ * @private
+ *
+ * @param {string?} roomId   room id for decryptor. If undefined, a temporary
+ * decryptor is instantiated.
+ *
+ * @param {string} algorithm  crypto algorithm
+ *
+ * @return {module:crypto.algorithms.base.DecryptionAlgorithm}
+ *
+ * @raises {module:crypto.algorithms.DecryptionError} if the algorithm is
+ * unknown
+ */
+Crypto.prototype._getRoomDecryptor = function(roomId, algorithm) {
+    var decryptors;
+    var alg;
+
+    roomId = roomId || null;
+    if (roomId) {
+        decryptors = this._roomDecryptors[roomId];
+        if (!decryptors) {
+            this._roomDecryptors[roomId] = decryptors = {};
        }
-    }).catch(function(e) {
-        console.error(
-            "Error updating device keys for new device " + userId + ":" +
-                deviceId,
-            e
+
+        alg = decryptors[algorithm];
+        if (alg) {
+            return alg;
+        }
+    }
+
+    var AlgClass = algorithms.DECRYPTION_CLASSES[algorithm];
+    if (!AlgClass) {
+        throw new algorithms.DecryptionError(
+            'Unknown encryption algorithm "' + algorithm + '".'
        );
-    }).done();
+    }
+    alg = new AlgClass({
+        userId: this._userId,
+        crypto: this,
+        olmDevice: this._olmDevice,
+        roomId: roomId,
+    });
+
+    if (decryptors) {
+        decryptors[algorithm] = alg;
+    }
+    return alg;
+};
+
+
+/**
+ * sign the given object with our ed25519 key
+ *
+ * @param {Object} obj  Object to which we will add a 'signatures' property
+ */
+Crypto.prototype._signObject = function(obj) {
+    var sigs = {};
+    sigs[this._userId] = {};
+    sigs[this._userId]["ed25519:" + this._deviceId] =
+        this._olmDevice.sign(anotherjson.stringify(obj));
+    obj.signatures = sigs;
 };

 /**
@@ -20,6 +20,9 @@ limitations under the License.
 * Utilities common to olm encryption algorithms
 */

+var q = require('q');
+var anotherjson = require('another-json');
+
 var utils = require("../utils");

 /**
@@ -34,22 +37,38 @@ module.exports.MEGOLM_ALGORITHM = "m.megolm.v1.aes-sha2";


 /**
- * Encrypt an event payload for a list of devices
+ * Encrypt an event payload for an Olm device
 *
+ * @param {Object<string, string>} resultsObject  The `ciphertext` property
+ *   of the m.room.encrypted event to which to add our result
+ *
+ * @param {string} ourUserId
 * @param {string} ourDeviceId
 * @param {module:crypto/OlmDevice} olmDevice olm.js wrapper
- * @param {string[]} participantKeys list of curve25519 keys to encrypt for
+ * @param {string} recipientUserId
+ * @param {module:crypto/deviceinfo} recipientDevice
 * @param {object} payloadFields fields to include in the encrypted payload
- *
- * @return {object} content for an m.room.encrypted event
 */
-module.exports.encryptMessageForDevices = function(
-    ourDeviceId, olmDevice, participantKeys, payloadFields
+module.exports.encryptMessageForDevice = function(
+    resultsObject,
+    ourUserId, ourDeviceId, olmDevice, recipientUserId, recipientDevice,
+    payloadFields
 ) {
-    participantKeys.sort();
-    var participantHash = ""; // Olm.sha256(participantKeys.join());
-    var payloadJson = {
-        fingerprint: participantHash,
+    var deviceKey = recipientDevice.getIdentityKey();
+    var sessionId = olmDevice.getSessionIdForDevice(deviceKey);
+    if (sessionId === null) {
+        // If we don't have a session for a device then
+        // we can't encrypt a message for it.
+        return;
+    }
+
+    console.log(
+        "Using sessionid " + sessionId + " for device " +
+            recipientUserId + ":" + recipientDevice.deviceId
+    );
+
+    var payload = {
+        sender: ourUserId,
        sender_device: ourDeviceId,

        // Include the Ed25519 key so that the recipient knows what
@@ -63,28 +82,187 @@ module.exports.encryptMessageForDevices = function(
        keys: {
            "ed25519": olmDevice.deviceEd25519Key,
        },
-    };
-    utils.extend(payloadJson, payloadFields);

-    var ciphertext = {};
-    var payloadString = JSON.stringify(payloadJson);
-    for (var i = 0; i < participantKeys.length; ++i) {
-        var deviceKey = participantKeys[i];
-        var sessionId = olmDevice.getSessionIdForDevice(deviceKey);
-        if (sessionId === null) {
-            // If we don't have a session for a device then
-            // we can't encrypt a message for it.
-            continue;
-        }
-        console.log("Using sessionid " + sessionId + " for device " + deviceKey);
-        ciphertext[deviceKey] = olmDevice.encryptMessage(
-            deviceKey, sessionId, payloadString
-        );
-    }
-    var encryptedContent = {
-        algorithm: module.exports.OLM_ALGORITHM,
-        sender_key: olmDevice.deviceCurve25519Key,
-        ciphertext: ciphertext
+        // include the recipient device details in the payload,
+        // to avoid unknown key attacks, per
+        // https://github.com/vector-im/vector-web/issues/2483
+        recipient: recipientUserId,
+        recipient_keys: {
+            "ed25519": recipientDevice.getFingerprint(),
+        },
    };
-    return encryptedContent;
+
+    // TODO: technically, a bunch of that stuff only needs to be included for
+    // pre-key messages: after that, both sides know exactly which devices are
+    // involved in the session. If we're looking to reduce data transfer in the
+    // future, we could elide them for subsequent messages.
+
+    utils.extend(payload, payloadFields);
+
+    resultsObject[deviceKey] = olmDevice.encryptMessage(
+        deviceKey, sessionId, JSON.stringify(payload)
+    );
+};
+
+/**
+ * Try to make sure we have established olm sessions for the given devices.
+ *
+ * @param {module:crypto/OlmDevice} olmDevice
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices
+ *
+ * @return {module:client.Promise} resolves once the sessions are complete, to
+ *    an Object mapping from userId to deviceId to
+ *    {@link module:crypto~OlmSessionResult}
+ */
+module.exports.ensureOlmSessionsForDevices = function(
+    olmDevice, baseApis, devicesByUser
+) {
+    var devicesWithoutSession = [
+        // [userId, deviceId], ...
+    ];
+    var result = {};
+
+    for (var userId in devicesByUser) {
+        if (!devicesByUser.hasOwnProperty(userId)) { continue; }
+        result[userId] = {};
+        var devices = devicesByUser[userId];
+        for (var j = 0; j < devices.length; j++) {
+            var deviceInfo = devices[j];
+            var deviceId = deviceInfo.deviceId;
+            var key = deviceInfo.getIdentityKey();
+            var sessionId = olmDevice.getSessionIdForDevice(key);
+            if (sessionId === null) {
+                devicesWithoutSession.push([userId, deviceId]);
+            }
+            result[userId][deviceId] = {
+                device: deviceInfo,
+                sessionId: sessionId,
+            };
+        }
+    }
+
+    if (devicesWithoutSession.length === 0) {
+        return q(result);
+    }
+
+    // TODO: this has a race condition - if we try to send another message
+    // while we are claiming a key, we will end up claiming two and setting up
+    // two sessions.
+    //
+    // That should eventually resolve itself, but it's poor form.
+
+    var oneTimeKeyAlgorithm = "signed_curve25519";
+    return baseApis.claimOneTimeKeys(
+        devicesWithoutSession, oneTimeKeyAlgorithm
+    ).then(function(res) {
+        for (var userId in devicesByUser) {
+            if (!devicesByUser.hasOwnProperty(userId)) { continue; }
+            var userRes = res.one_time_keys[userId] || {};
+            var devices = devicesByUser[userId];
+            for (var j = 0; j < devices.length; j++) {
+                var deviceInfo = devices[j];
+                var deviceId = deviceInfo.deviceId;
+                if (result[userId][deviceId].sessionId) {
+                    // we already have a result for this device
+                    continue;
+                }
+
+                var deviceRes = userRes[deviceId] || {};
+                var oneTimeKey = null;
+                for (var keyId in deviceRes) {
+                    if (keyId.indexOf(oneTimeKeyAlgorithm + ":") === 0) {
+                        oneTimeKey = deviceRes[keyId];
+                    }
+                }
+
+                if (!oneTimeKey) {
+                    console.warn(
+                        "No one-time keys (alg=" + oneTimeKeyAlgorithm +
+                            ") for device " + userId + ":" + deviceId
+                    );
+                    continue;
+                }
+
+                var sid = _verifyKeyAndStartSession(
+                    olmDevice, oneTimeKey, userId, deviceInfo
+                );
+                result[userId][deviceId].sessionId = sid;
+            }
+        }
+        return result;
+    });
+};
+
+
+function _verifyKeyAndStartSession(olmDevice, oneTimeKey, userId, deviceInfo) {
+    var deviceId = deviceInfo.deviceId;
+    try {
+        _verifySignature(
+            olmDevice, oneTimeKey, userId, deviceId,
+            deviceInfo.getFingerprint()
+        );
+    } catch (e) {
+        console.error(
+            "Unable to verify signature on one-time key for device " +
+                userId + ":" + deviceId + ":", e
+        );
+        return null;
+    }
+
+    var sid;
+    try {
+        sid = olmDevice.createOutboundSession(
+            deviceInfo.getIdentityKey(), oneTimeKey.key
+        );
+    } catch (e) {
+        // possibly a bad key
+        console.error("Error starting session with device " +
+                      userId + ":" + deviceId + ": " + e);
+        return null;
+    }
+
+    console.log("Started new sessionid " + sid +
+                " for device " + userId + ":" + deviceId);
+    return sid;
+}
+
+
+/**
+ * Verify the signature on an object
+ *
+ * @param {module:crypto/OlmDevice} olmDevice olm wrapper to use for verify op
+ *
+ * @param {Object} obj object to check signature on. Note that this will be
+ * stripped of its 'signatures' and 'unsigned' properties.
+ *
+ * @param {string} signingUserId  ID of the user whose signature should be checked
+ *
+ * @param {string} signingDeviceId  ID of the device whose signature should be checked
+ *
+ * @param {string} signingKey   base64-ed ed25519 public key
+ */
+var _verifySignature = module.exports.verifySignature = function(
+    olmDevice, obj, signingUserId, signingDeviceId, signingKey
+) {
+    var signKeyId = "ed25519:" + signingDeviceId;
+    var signatures = obj.signatures || {};
+    var userSigs = signatures[signingUserId] || {};
+    var signature = userSigs[signKeyId];
+    if (!signature) {
+        throw Error("No signature");
+    }
+
+    // prepare the canonical json: remove unsigned and signatures, and stringify with
+    // anotherjson
+    delete obj.unsigned;
+    delete obj.signatures;
+    var json = anotherjson.stringify(obj);
+
+    olmDevice.verifySignature(
+        signingKey, json, signature
+    );
 };
@@ -63,9 +63,11 @@ module.exports.PREFIX_MEDIA_R0 = "/_matrix/media/r0";
 * requests. This function must look like function(opts, callback){ ... }.
 * @param {string} opts.prefix Required. The matrix client prefix to use, e.g.
 * '/_matrix/client/r0'. See PREFIX_R0 and PREFIX_UNSTABLE for constants.
- * @param {bool} opts.onlyData True to return only the 'data' component of the
- * response (e.g. the parsed HTTP body). If false, requests will return status
- * codes and headers in addition to data. Default: false.
+ *
+ * @param {bool=} opts.onlyData True to return only the 'data' component of the
+ * response (e.g. the parsed HTTP body). If false, requests will return an
+ * object with the properties <tt>code</tt>, <tt>headers</tt> and <tt>data</tt>.
+ *
 * @param {string} opts.accessToken The access_token to send with requests. Can be
 * null to not send an access token.
 * @param {Object} opts.extraParams Optional. Extra query parameters to send on
@@ -99,22 +101,87 @@ module.exports.MatrixHttpApi.prototype = {

    /**
     * Upload content to the Home Server
-     * @param {File} file A File object (in a browser) or in Node,
-                                 an object with properties:
-                                 name: The file's name
-                                 stream: A read stream
-     * @param {Function} callback Optional. The callback to invoke on
-     * success/failure. See the promise return values for more information.
-     * @return {module:client.Promise} Resolves to <code>{data: {Object},
+     *
+     * @param {object} file The object to upload. On a browser, something that
+     *   can be sent to XMLHttpRequest.send (typically a File).  Under node.js,
+     *   a Buffer, String or ReadStream.
+     *
+     * @param {object} opts  options object
+     *
+     * @param {string=} opts.name   Name to give the file on the server. Defaults
+     *   to <tt>file.name</tt>.
+     *
+     * @param {string=} opts.type   Content-type for the upload. Defaults to
+     *   <tt>file.type</tt>, or <tt>applicaton/octet-stream</tt>.
+     *
+     * @param {boolean=} opts.rawResponse Return the raw body, rather than
+     *   parsing the JSON. Defaults to false (except on node.js, where it
+     *   defaults to true for backwards compatibility).
+     *
+     * @param {boolean=} opts.onlyContentUri Just return the content URI,
+     *   rather than the whole body. Defaults to false (except on browsers,
+     *   where it defaults to true for backwards compatibility). Ignored if
+     *   opts.rawResponse is true.
+     *
+     * @param {Function=} opts.callback Deprecated. Optional. The callback to
+     *    invoke on success/failure. See the promise return values for more
+     *    information.
+     *
+     * @return {module:client.Promise} Resolves to response object, as
+     *    determined by this.opts.onlyData, opts.rawResponse, and
+     *    opts.onlyContentUri.  Rejects with an error (usually a MatrixError).
     */
-    uploadContent: function(file, callback) {
-        if (callback !== undefined && !utils.isFunction(callback)) {
-            throw Error(
-                "Expected callback to be a function but got " + typeof callback
-            );
+    uploadContent: function(file, opts) {
+        if (utils.isFunction(opts)) {
+            // opts used to be callback
+            opts = {
+                callback: opts,
+            };
+        } else if (opts === undefined) {
+            opts = {};
        }
-        var defer = q.defer();
-        var url = this.opts.baseUrl + "/_matrix/media/v1/upload";
+
+        // if the file doesn't have a mime type, use a default since
+        // the HS errors if we don't supply one.
+        var contentType = opts.type || file.type || 'application/octet-stream';
+        var fileName = opts.name || file.name;
+
+        // we used to recommend setting file.stream to the thing to upload on
+        // nodejs.
+        var body = file.stream ? file.stream : file;
+
+        // backwards-compatibility hacks where we used to do different things
+        // between browser and node.
+        var rawResponse = opts.rawResponse;
+        if (rawResponse === undefined) {
+            if (global.XMLHttpRequest) {
+                rawResponse = false;
+            } else {
+                console.warn(
+                    "Returning the raw JSON from uploadContent(). Future " +
+                    "versions of the js-sdk will change this default, to " +
+                    "return the parsed object. Set opts.rawResponse=false " +
+                    "to change this behaviour now."
+                );
+                rawResponse = true;
+            }
+        }
+
+        var onlyContentUri = opts.onlyContentUri;
+        if (!rawResponse && onlyContentUri === undefined) {
+            if (global.XMLHttpRequest) {
+                console.warn(
+                    "Returning only the content-uri from uploadContent(). " +
+                    "Future versions of the js-sdk will change this " +
+                    "default, to return the whole response object. Set " +
+                    "opts.onlyContentUri=false to change this behaviour now."
+                );
+                onlyContentUri = true;
+            } else {
+                onlyContentUri = false;
+            }
+        }
+
        // browser-request doesn't support File objects because it deep-copies
        // the options using JSON.parse(JSON.stringify(options)). Instead of
        // loading the whole file into memory as a string and letting
@@ -124,40 +191,60 @@ module.exports.MatrixHttpApi.prototype = {
        // of important here)

        var upload = { loaded: 0, total: 0 };
+        var promise;
+
+        // XMLHttpRequest doesn't parse JSON for us. request normally does, but
+        // we're setting opts.json=false so that it doesn't JSON-encode the
+        // request, which also means it doesn't JSON-decode the response. Either
+        // way, we have to JSON-parse the response ourselves.
+        var bodyParser = null;
+        if (!rawResponse) {
+            bodyParser = function(rawBody) {
+                var body = JSON.parse(rawBody);
+                if (onlyContentUri) {
+                    body = body.content_uri;
+                    if (body === undefined) {
+                        throw Error('Bad response');
+                    }
+                }
+                return body;
+            };
+        }

        if (global.XMLHttpRequest) {
+            var defer = q.defer();
            var xhr = new global.XMLHttpRequest();
            upload.xhr = xhr;
-            var cb = requestCallback(defer, callback, this.opts.onlyData);
+            var cb = requestCallback(defer, opts.callback, this.opts.onlyData);

            var timeout_fn = function() {
                xhr.abort();
                cb(new Error('Timeout'));
            };

+            // set an initial timeout of 30s; we'll advance it each time we get
+            // a progress notification
            xhr.timeout_timer = callbacks.setTimeout(timeout_fn, 30000);

            xhr.onreadystatechange = function() {
                switch (xhr.readyState) {
                    case global.XMLHttpRequest.DONE:
                        callbacks.clearTimeout(xhr.timeout_timer);
-                        var err;
-                        if (!xhr.responseText) {
-                            err = new Error('No response body.');
+                        var resp;
+                        try {
+                            if (!xhr.responseText) {
+                                throw new Error('No response body.');
+                            }
+                            resp = xhr.responseText;
+                            if (bodyParser) {
+                                resp = bodyParser(resp);
+                            }
+                        } catch (err) {
                            err.http_status = xhr.status;
                            cb(err);
                            return;
                        }
-
-                        var resp = JSON.parse(xhr.responseText);
-                        if (resp.content_uri === undefined) {
-                            err = Error('Bad response');
-                            err.http_status = xhr.status;
-                            cb(err);
-                            return;
-                        }
-
-                        cb(undefined, xhr, resp.content_uri);
+                        cb(undefined, xhr, resp);
                        break;
                }
            };
@@ -168,59 +255,57 @@ module.exports.MatrixHttpApi.prototype = {
                xhr.timeout_timer = callbacks.setTimeout(timeout_fn, 30000);
                defer.notify(ev);
            });
+            var url = this.opts.baseUrl + "/_matrix/media/v1/upload";
            url += "?access_token=" + encodeURIComponent(this.opts.accessToken);
-            url += "&filename=" + encodeURIComponent(file.name);
+            url += "&filename=" + encodeURIComponent(fileName);

            xhr.open("POST", url);
-            if (file.type) {
-                xhr.setRequestHeader("Content-Type", file.type);
-            } else {
-                // if the file doesn't have a mime type, use a default since
-                // the HS errors if we don't supply one.
-                xhr.setRequestHeader("Content-Type", 'application/octet-stream');
-            }
-            xhr.send(file);
+            xhr.setRequestHeader("Content-Type", contentType);
+            xhr.send(body);
+            promise = defer.promise;
+
+            // dirty hack (as per _request) to allow the upload to be cancelled.
+            promise.abort = xhr.abort.bind(xhr);
        } else {
            var queryParams = {
-                filename: file.name,
-                access_token: this.opts.accessToken
+                filename: fileName,
            };
-            upload.request = this.opts.request({
-                uri: url,
-                qs: queryParams,
-                method: "POST",
-                headers: {"Content-Type": file.type},
-                body: file.stream
-            }, requestCallback(defer, callback, this.opts.onlyData));
+
+            promise = this.authedRequest(
+                opts.callback, "POST", "/upload", queryParams, body, {
+                    prefix: "/_matrix/media/v1",
+                    headers: {"Content-Type": contentType},
+                    json: false,
+                    bodyParser: bodyParser,
+                }
+            );
        }

-        this.uploads.push(upload);
-
        var self = this;
-        upload.promise = defer.promise.finally(function() {
-            var uploadsKeys = Object.keys(self.uploads);
-            for (var i = 0; i < uploadsKeys.length; ++i) {
-                if (self.uploads[uploadsKeys[i]].promise === defer.promise) {
-                    self.uploads.splice(uploadsKeys[i], 1);
+
+        // remove the upload from the list on completion
+        var promise0 = promise.finally(function() {
+            for (var i = 0; i < self.uploads.length; ++i) {
+                if (self.uploads[i] === upload) {
+                    self.uploads.splice(i, 1);
+                    return;
                }
            }
        });
-        return upload.promise;
+
+        // copy our dirty abort() method to the new promise
+        promise0.abort = promise.abort;
+
+        upload.promise = promise0;
+        this.uploads.push(upload);
+
+        return promise0;
    },

    cancelUpload: function(promise) {
-        var uploadsKeys = Object.keys(this.uploads);
-        for (var i = 0; i < uploadsKeys.length; ++i) {
-            var upload = this.uploads[uploadsKeys[i]];
-            if (upload.promise === promise) {
-                if (upload.xhr !== undefined) {
-                    upload.xhr.abort();
-                    return true;
-                } else if (upload.request !== undefined) {
-                    upload.request.abort();
-                    return true;
-                }
-            }
+        if (promise.abort) {
+            promise.abort();
+            return true;
        }
        return false;
    },
@@ -271,11 +356,22 @@ module.exports.MatrixHttpApi.prototype = {
     * @param {string} method The HTTP method e.g. "GET".
     * @param {string} path The HTTP path <b>after</b> the supplied prefix e.g.
     * "/createRoom".
-     * @param {Object} queryParams A dict of query params (these will NOT be
-     * urlencoded).
+     *
+     * @param {Object=} queryParams A dict of query params (these will NOT be
+     * urlencoded). If unspecified, there will be no query params.
+     *
     * @param {Object} data The HTTP JSON body.
-     * @param {Number=} localTimeoutMs The maximum amount of time to wait before
+     *
+     * @param {Object=} opts additional options
+     *
+     * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
     * timing out the request. If not specified, there is no timeout.
+     *
+     * @param {sting=} opts.prefix The full prefix to use e.g.
+     * "/_matrix/client/v2_alpha". If not specified, uses this.opts.prefix.
+     *
+     * @param {Object=} opts.headers map of additional request headers
+     *
     * @return {module:client.Promise} Resolves to <code>{data: {Object},
     * headers: {Object}, code: {Number}}</code>.
     * If <code>onlyData</code> is set, this will resolve to the <code>data</code>
@@ -283,18 +379,25 @@ module.exports.MatrixHttpApi.prototype = {
     * @return {module:http-api.MatrixError} Rejects with an error if a problem
     * occurred. This includes network problems and Matrix-specific error JSON.
     */
-    authedRequest: function(callback, method, path, queryParams, data, localTimeoutMs) {
-        if (!queryParams) { queryParams = {}; }
-        queryParams.access_token = this.opts.accessToken;
-        var self = this;
+    authedRequest: function(callback, method, path, queryParams, data, opts) {
+        if (!queryParams) {
+            queryParams = {};
+        }
+        if (!queryParams.access_token) {
+            queryParams.access_token = this.opts.accessToken;
+        }
+
        var request_promise = this.request(
-            callback, method, path, queryParams, data, localTimeoutMs
+            callback, method, path, queryParams, data, opts
        );
+
+        var self = this;
        request_promise.catch(function(err) {
            if (err.errcode == 'M_UNKNOWN_TOKEN') {
                self.event_emitter.emit("Session.logged_out");
            }
        });
+
        // return the original promise, otherwise tests break due to it having to
        // go around the event loop one more time to process the result of the request
        return request_promise;
@@ -307,11 +410,22 @@ module.exports.MatrixHttpApi.prototype = {
     * @param {string} method The HTTP method e.g. "GET".
     * @param {string} path The HTTP path <b>after</b> the supplied prefix e.g.
     * "/createRoom".
-     * @param {Object} queryParams A dict of query params (these will NOT be
-     * urlencoded).
+     *
+     * @param {Object=} queryParams A dict of query params (these will NOT be
+     * urlencoded). If unspecified, there will be no query params.
+     *
     * @param {Object} data The HTTP JSON body.
-     * @param {Number=} localTimeoutMs The maximum amount of time to wait before
+     *
+     * @param {Object=} opts additional options
+     *
+     * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
     * timing out the request. If not specified, there is no timeout.
+     *
+     * @param {sting=} opts.prefix The full prefix to use e.g.
+     * "/_matrix/client/v2_alpha". If not specified, uses this.opts.prefix.
+     *
+     * @param {Object=} opts.headers map of additional request headers
+     *
     * @return {module:client.Promise} Resolves to <code>{data: {Object},
     * headers: {Object}, code: {Number}}</code>.
     * If <code>onlyData</code> is set, this will resolve to the <code>data</code>
@@ -319,9 +433,13 @@ module.exports.MatrixHttpApi.prototype = {
     * @return {module:http-api.MatrixError} Rejects with an error if a problem
     * occurred. This includes network problems and Matrix-specific error JSON.
     */
-    request: function(callback, method, path, queryParams, data, localTimeoutMs) {
-        return this.requestWithPrefix(
-            callback, method, path, queryParams, data, this.opts.prefix, localTimeoutMs
+    request: function(callback, method, path, queryParams, data, opts) {
+        opts = opts || {};
+        var prefix = opts.prefix !== undefined ? opts.prefix : this.opts.prefix;
+        var fullUri = this.opts.baseUrl + prefix + path;
+
+        return this.requestOtherUrl(
+            callback, method, fullUri, queryParams, data, opts
        );
    },

@@ -347,16 +465,16 @@ module.exports.MatrixHttpApi.prototype = {
     * object only.
     * @return {module:http-api.MatrixError} Rejects with an error if a problem
     * occurred. This includes network problems and Matrix-specific error JSON.
+     *
+     * @deprecated prefer authedRequest with opts.prefix
     */
    authedRequestWithPrefix: function(callback, method, path, queryParams, data,
                                      prefix, localTimeoutMs) {
-        var fullUri = this.opts.baseUrl + prefix + path;
-        if (!queryParams) {
-            queryParams = {};
-        }
-        queryParams.access_token = this.opts.accessToken;
-        return this._request(
-            callback, method, fullUri, queryParams, data, localTimeoutMs
+        return this.authedRequest(
+            callback, method, path, queryParams, data, {
+                localTimeoutMs: localTimeoutMs,
+                prefix: prefix,
+            }
        );
    },

@@ -382,15 +500,16 @@ module.exports.MatrixHttpApi.prototype = {
     * object only.
     * @return {module:http-api.MatrixError} Rejects with an error if a problem
     * occurred. This includes network problems and Matrix-specific error JSON.
+     *
+     * @deprecated prefer request with opts.prefix
     */
    requestWithPrefix: function(callback, method, path, queryParams, data, prefix,
                                localTimeoutMs) {
-        var fullUri = this.opts.baseUrl + prefix + path;
-        if (!queryParams) {
-            queryParams = {};
-        }
-        return this._request(
-            callback, method, fullUri, queryParams, data, localTimeoutMs
+        return this.request(
+            callback, method, path, queryParams, data, {
+                localTimeoutMs: localTimeoutMs,
+                prefix: prefix,
+            }
        );
    },

@@ -400,11 +519,22 @@ module.exports.MatrixHttpApi.prototype = {
     * success/failure. See the promise return values for more information.
     * @param {string} method The HTTP method e.g. "GET".
     * @param {string} uri The HTTP URI
-     * @param {Object} queryParams A dict of query params (these will NOT be
-     * urlencoded).
+     *
+     * @param {Object=} queryParams A dict of query params (these will NOT be
+     * urlencoded). If unspecified, there will be no query params.
+     *
     * @param {Object} data The HTTP JSON body.
-     * @param {Number=} localTimeoutMs The maximum amount of time to wait before
+     *
+     * @param {Object=} opts additional options
+     *
+     * @param {Number=} opts.localTimeoutMs The maximum amount of time to wait before
     * timing out the request. If not specified, there is no timeout.
+     *
+     * @param {sting=} opts.prefix The full prefix to use e.g.
+     * "/_matrix/client/v2_alpha". If not specified, uses this.opts.prefix.
+     *
+     * @param {Object=} opts.headers map of additional request headers
+     *
     * @return {module:client.Promise} Resolves to <code>{data: {Object},
     * headers: {Object}, code: {Number}}</code>.
     * If <code>onlyData</code> is set, this will resolve to the <code>data</code>
@@ -413,12 +543,18 @@ module.exports.MatrixHttpApi.prototype = {
     * occurred. This includes network problems and Matrix-specific error JSON.
     */
    requestOtherUrl: function(callback, method, uri, queryParams, data,
-                                localTimeoutMs) {
-        if (!queryParams) {
-            queryParams = {};
+                              opts) {
+        if (opts === undefined || opts === null) {
+            opts = {};
+        } else if (isFinite(opts)) {
+            // opts used to be localTimeoutMs
+            opts = {
+                localTimeoutMs: opts
+            };
        }
+
        return this._request(
-            callback, method, uri, queryParams, data, localTimeoutMs
+            callback, method, uri, queryParams, data, opts
        );
    },

@@ -441,27 +577,56 @@ module.exports.MatrixHttpApi.prototype = {
        return this.opts.baseUrl + prefix + path + queryString;
    },

-    _request: function(callback, method, uri, queryParams, data, localTimeoutMs) {
+    /**
+     * @private
+     *
+     * @param {function} callback
+     * @param {string} method
+     * @param {string} uri
+     * @param {object} queryParams
+     * @param {object|string} data
+     * @param {object=} opts
+     *
+     * @param {boolean} [opts.json =true] Json-encode data before sending, and
+     *   decode response on receipt. (We will still json-decode error
+     *   responses, even if this is false.)
+     *
+     * @param {object=} opts.headers  extra request headers
+     *
+     * @param {number=} opts.localTimeoutMs client-side timeout for the
+     *    request. No timeout if undefined.
+     *
+     * @param {function=} opts.bodyParser function to parse the body of the
+     *    response before passing it to the promise and callback.
+     *
+     * @return {module:client.Promise} a promise which resolves to either the
+     * response object (if this.opts.onlyData is truthy), or the parsed
+     * body. Rejects
+     */
+    _request: function(callback, method, uri, queryParams, data, opts) {
        if (callback !== undefined && !utils.isFunction(callback)) {
            throw Error(
                "Expected callback to be a function but got " + typeof callback
            );
        }
+        opts = opts || {};
+
        var self = this;
-        if (!queryParams) {
-            queryParams = {};
-        }
        if (this.opts.extraParams) {
            for (var key in this.opts.extraParams) {
                if (!this.opts.extraParams.hasOwnProperty(key)) { continue; }
                queryParams[key] = this.opts.extraParams[key];
            }
        }
+
+        var json = opts.json === undefined ? true : opts.json;
+
        var defer = q.defer();

        var timeoutId;
        var timedOut = false;
        var req;
+        var localTimeoutMs = opts.localTimeoutMs;
        if (localTimeoutMs) {
            timeoutId = callbacks.setTimeout(function() {
                timedOut = true;
@@ -486,8 +651,9 @@ module.exports.MatrixHttpApi.prototype = {
                    withCredentials: false,
                    qs: queryParams,
                    body: data,
-                    json: true,
+                    json: json,
                    timeout: localTimeoutMs,
+                    headers: opts.headers || {},
                    _matrix_opts: this.opts
                },
                function(err, response, body) {
@@ -497,7 +663,15 @@ module.exports.MatrixHttpApi.prototype = {
                            return; // already rejected promise
                        }
                    }
-                    var handlerFn = requestCallback(defer, callback, self.opts.onlyData);
+
+                    // if json is falsy, we won't parse any error response, so need
+                    // to do so before turning it into a MatrixError
+                    var parseErrorJson = !json;
+                    var handlerFn = requestCallback(
+                        defer, callback, self.opts.onlyData,
+                        parseErrorJson,
+                        opts.bodyParser
+                    );
                    handlerFn(err, response, body);
                }
            );
@@ -524,14 +698,34 @@ module.exports.MatrixHttpApi.prototype = {
 *
 * If onlyData is true, the defer/callback is invoked with the body of the
 * response, otherwise the result code.
+ *
+ * If parseErrorJson is true, we will JSON.parse the body if we get a 4xx error.
+ *
 */
-var requestCallback = function(defer, userDefinedCallback, onlyData) {
+var requestCallback = function(
+    defer, userDefinedCallback, onlyData,
+    parseErrorJson, bodyParser
+) {
    userDefinedCallback = userDefinedCallback || function() {};

    return function(err, response, body) {
-        if (!err && response.statusCode >= 400) {
-            err = new module.exports.MatrixError(body);
-            err.httpStatus = response.statusCode;
+        if (!err) {
+            try {
+                if (response.statusCode >= 400) {
+                    if (parseErrorJson) {
+                        // we won't have json-decoded the response.
+                        body = JSON.parse(body);
+                    }
+                    err = new module.exports.MatrixError(body);
+                } else if (bodyParser) {
+                    body = bodyParser(body);
+                }
+            } catch (e) {
+                err = e;
+            }
+            if (err) {
+                err.httpStatus = response.statusCode;
+            }
        }

        if (err) {
@@ -0,0 +1,228 @@
+/*
+Copyright 2016 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+/** @module interactive-auth */
+var q = require("q");
+
+var utils = require("./utils");
+
+/**
+ * Abstracts the logic used to drive the interactive auth process.
+ *
+ * <p>Components implementing an interactive auth flow should instantiate one of
+ * these, passing in the necessary callbacks to the constructor. They should
+ * then call attemptAuth, which will return a promise which will resolve or
+ * reject when the interactive-auth process completes.
+ *
+ * <p>Meanwhile, calls will be made to the startAuthStage and doRequest
+ * callbacks, and information gathered from the user can be submitted with
+ * submitAuthDict.
+ *
+ * @constructor
+ * @alias module:interactive-auth
+ *
+ * @param {object} opts  options object
+ *
+ * @param {object?} opts.authData error response from the last request. If
+ *    null, a request will be made with no auth before starting.
+ *
+ * @param {function(object?): module:client.Promise} opts.doRequest
+ *     called with the new auth dict to submit the request. Should return a
+ *     promise which resolves to the successful response or rejects with a
+ *     MatrixError.
+ *
+ * @param {function(string, object?)} opts.startAuthStage
+ *     called to ask the UI to start a particular auth stage. The arguments
+ *     are: the login type (eg m.login.password); and (if the last request
+ *     returned an error), an error object, with fields 'errcode' and 'error'.
+ *
+ */
+function InteractiveAuth(opts) {
+    this._data = opts.authData;
+    this._requestCallback = opts.doRequest;
+    this._startAuthStageCallback = opts.startAuthStage;
+    this._completionDeferred = null;
+}
+
+InteractiveAuth.prototype = {
+    /**
+     * begin the authentication process.
+     *
+     * @return {module:client.Promise}  which resolves to the response on success,
+     * or rejects with the error on failure.
+     */
+    attemptAuth: function() {
+        this._completionDeferred = q.defer();
+
+        if (!this._data) {
+            this._doRequest(null);
+        } else {
+            this._startNextAuthStage();
+        }
+
+        return this._completionDeferred.promise;
+    },
+
+    /**
+     * get the auth session ID
+     *
+     * @return {string} session id
+     */
+    getSessionId: function() {
+        return this._data ? this._data.session : undefined;
+    },
+
+    /**
+     * get the server params for a given stage
+     *
+     * @param {string}  login type for the stage
+     * @return {object?}  any parameters from the server for this stage
+     */
+    getStageParams: function(loginType) {
+        var params = {};
+        if (this._data && this._data.params) {
+            params = this._data.params;
+        }
+        return params[loginType];
+    },
+
+    /**
+     * submit a new auth dict and fire off the request. This will either
+     * make attemptAuth resolve/reject, or cause the startAuthStage callback
+     * to be called for a new stage.
+     *
+     * @param {object} authData new auth dict to send to the server. Should
+     *    include a `type` propterty denoting the login type, as well as any
+     *    other params for that stage.
+     */
+    submitAuthDict: function(authData) {
+        if (!this._completionDeferred) {
+            throw new Error("submitAuthDict() called before attemptAuth()");
+        }
+
+        // use the sessionid from the last request.
+        var auth = {
+            session: this._data.session,
+        };
+        utils.extend(auth, authData);
+
+        this._doRequest(auth);
+    },
+
+    /**
+     * Fire off a request, and either resolve the promise, or call
+     * startAuthStage.
+     *
+     * @private
+     * @param {object?} auth new auth dict, including session id
+     */
+    _doRequest: function(auth) {
+        var self = this;
+
+        // hackery to make sure that synchronous exceptions end up in the catch
+        // handler (without the additional event loop entailed by q.fcall or an
+        // extra q().then)
+        var prom;
+        try {
+            prom = this._requestCallback(auth);
+        } catch (e) {
+            prom = q.reject(e);
+        }
+
+        prom.then(
+            function(result) {
+                console.log("result from request: ", result);
+                self._completionDeferred.resolve(result);
+            }, function(error) {
+                if (error.httpStatus !== 401 || !error.data || !error.data.flows) {
+                    // doesn't look like an interactive-auth failure. fail the whole lot.
+                    throw error;
+                }
+                self._data = error.data;
+                self._startNextAuthStage();
+            }
+        ).catch(this._completionDeferred.reject).done();
+    },
+
+    /**
+     * Pick the next stage and call the callback
+     *
+     * @private
+     */
+    _startNextAuthStage: function() {
+        var nextStage = this._chooseStage();
+        if (!nextStage) {
+            throw new Error("No incomplete flows from the server");
+        }
+
+        var stageError = null;
+        if (this._data.errcode || this._data.error) {
+            stageError = {
+                errcode: this._data.errcode || "",
+                error: this._data.error || "",
+            };
+        }
+        this._startAuthStageCallback(nextStage, stageError);
+    },
+
+    /**
+     * Pick the next auth stage
+     *
+     * @private
+     * @return {string?} login type
+     */
+    _chooseStage: function() {
+        var flow = this._chooseFlow();
+        console.log("Active flow => %s", JSON.stringify(flow));
+        var nextStage = this._firstUncompletedStage(flow);
+        console.log("Next stage: %s", nextStage);
+        return nextStage;
+    },
+
+    /**
+     * Pick one of the flows from the returned list
+     *
+     * @private
+     * @return {object} flow
+     */
+    _chooseFlow: function() {
+        var flows = this._data.flows || [];
+        // always use the first flow for now
+        return flows[0];
+    },
+
+    /**
+     * Get the first uncompleted stage in the given flow
+     *
+     * @private
+     * @param {object} flow
+     * @return {string} login type
+     */
+    _firstUncompletedStage: function(flow) {
+        var completed = (this._data || {}).completed || [];
+        for (var i = 0; i < flow.stages.length; ++i) {
+            var stageType = flow.stages[i];
+            if (completed.indexOf(stageType) === -1) {
+                return stageType;
+            }
+        }
+    },
+};
+
+
+/** */
+module.exports = InteractiveAuth;
@@ -55,6 +55,9 @@ module.exports.ContentRepo = require("./content-repo");
 module.exports.Filter = require("./filter");
 /** The {@link module:timeline-window~TimelineWindow} class. */
 module.exports.TimelineWindow = require("./timeline-window").TimelineWindow;
+/** The {@link module:interactive-auth} class. */
+module.exports.InteractiveAuth = require("./interactive-auth");
+

 /**
 * Create a new Matrix Call.
@@ -79,6 +82,27 @@ module.exports.request = function(r) {
    request = r;
 };

+/**
+ * Return the currently-set request function.
+ * @return {requestFunction} The current request function.
+ */
+module.exports.getRequest = function() {
+    return request;
+};
+
+/**
+ * Apply wrapping code around the request function. The wrapper function is
+ * installed as the new request handler, and when invoked it is passed the
+ * previous value, along with the options and callback arguments.
+ * @param {requestWrapperFunction} wrapper The wrapping function.
+ */
+module.exports.wrapRequest = function(wrapper) {
+    var origRequest = request;
+    request = function(options, callback) {
+        return wrapper(origRequest, options, callback);
+    };
+};
+
 /**
 * Construct a Matrix Client. Similar to {@link module:client~MatrixClient}
 * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -126,6 +150,16 @@ module.exports.createClient = function(opts) {
 * @param {requestCallback} callback The request callback.
 */

+/**
+ * A wrapper for the request function interface.
+ * @callback requestWrapperFunction
+ * @param {requestFunction} origRequest The underlying request function being
+ * wrapped
+ * @param {Object} opts The options for this HTTP request, given in the same
+ * form as {@link requestFunction}.
+ * @param {requestCallback} callback The request callback.
+ */
+
 /**
  * The request callback interface for performing HTTP requests. This matches the
  * API for the {@link https://github.com/request/request#requestoptions-callback|
@@ -21,6 +21,10 @@ limitations under the License.
 * @module models/event
 */

+var EventEmitter = require("events").EventEmitter;
+
+var utils = require('../utils.js');
+
 /**
 * Enum for event statuses.
 * @readonly
@@ -51,15 +55,6 @@ module.exports.EventStatus = {
 *
 * @param {Object} event The raw event to be wrapped in this DAO
 *
- * @param {Object=} clearEvent For encrypted events, the plaintext payload for
- * the event (typically containing <tt>type</tt> and <tt>content</tt> fields).
- *
- * @param {Object=} keysProved Keys owned by the sender of this event.
- *    See {@link module:models/event.MatrixEvent#getKeysProved}.
- *
- * @param {Object=} keysClaimed Keys the sender of this event claims.
- *    See {@link module:models/event.MatrixEvent#getKeysClaimed}.
- *
 * @prop {Object} event The raw (possibly encrypted) event. <b>Do not access
 * this property</b> directly unless you absolutely have to. Prefer the getter
 * methods defined on this class. Using the getter methods shields your app
@@ -75,22 +70,23 @@ module.exports.EventStatus = {
 * Default: true. <strong>This property is experimental and may change.</strong>
 */
 module.exports.MatrixEvent = function MatrixEvent(
-    event, clearEvent, keysProved, keysClaimed
+    event
 ) {
    this.event = event || {};
    this.sender = null;
    this.target = null;
    this.status = null;
    this.forwardLooking = true;
-
-    this._clearEvent = clearEvent || {};
    this._pushActions = null;

-    this._keysProved = keysProved || {};
-    this._keysClaimed = keysClaimed || {};
+    this._clearEvent = {};
+    this._keysProved = {};
+    this._keysClaimed = {};
 };
+utils.inherits(module.exports.MatrixEvent, EventEmitter);

-module.exports.MatrixEvent.prototype = {
+
+utils.extend(module.exports.MatrixEvent.prototype, {

    /**
     * Get the event_id for this event.
@@ -239,12 +235,37 @@ module.exports.MatrixEvent.prototype = {
        this._keysClaimed = keys;
    },

+    /**
+     * Update the cleartext data on this event.
+     *
+     * (This is used after decrypting an event; it should not be used by applications).
+     *
+     * @internal
+     *
+     * @fires module:models/event.MatrixEvent#"Event.decrypted"
+     *
+     * @param {Object} clearEvent The plaintext payload for the event
+     *     (typically containing <tt>type</tt> and <tt>content</tt> fields).
+     *
+     * @param {Object=} keysProved Keys owned by the sender of this event.
+     *    See {@link module:models/event.MatrixEvent#getKeysProved}.
+     *
+     * @param {Object=} keysClaimed Keys the sender of this event claims.
+     *    See {@link module:models/event.MatrixEvent#getKeysClaimed}.
+     */
+    setClearData: function(clearEvent, keysProved, keysClaimed) {
+        this._clearEvent = clearEvent;
+        this._keysProved = keysProved || {};
+        this._keysClaimed = keysClaimed || {};
+        this.emit("Event.decrypted", this);
+    },
+
    /**
     * Check if the event is encrypted.
     * @return {boolean} True if this event is encrypted.
     */
    isEncrypted: function() {
-        return Boolean(this._clearEvent.type);
+        return this.event.type === "m.room.encrypted";
    },

    /**
@@ -353,7 +374,7 @@ module.exports.MatrixEvent.prototype = {
     setPushActions: function(pushActions) {
        this._pushActions = pushActions;
     },
-};
+});


 /* http://matrix.org/docs/spec/r0.0.1/client_server.html#redactions says:
@@ -394,3 +415,15 @@ var _REDACT_KEEP_CONTENT_MAP = {
                           },
    'm.room.aliases': {'aliases': 1},
 };
+
+
+
+
+/**
+ * Fires when an event is decrypted
+ *
+ * @event module:models/event.MatrixEvent#"Event.decrypted"
+ *
+ * @param {module:models/event.MatrixEvent} event
+ *    The matrix event which has been decrypted
+ */
@@ -60,13 +60,14 @@ function debuglog() {
 function SyncApi(client, opts) {
    this.client = client;
    opts = opts || {};
-    opts.initialSyncLimit = opts.initialSyncLimit || 8;
+    opts.initialSyncLimit = (
+        opts.initialSyncLimit === undefined ? 8 : opts.initialSyncLimit
+    );
    opts.resolveInvitesToProfiles = opts.resolveInvitesToProfiles || false;
    opts.pollTimeout = opts.pollTimeout || (30 * 1000);
    opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";
    this.opts = opts;
    this._peekRoomId = null;
-    this._syncConnectionLost = false;
    this._currentSyncRequest = null;
    this._syncState = null;
    this._running = false;
@@ -392,8 +393,13 @@ SyncApi.prototype.sync = function() {
    }

    function getFilter() {
-        var filter = new Filter(client.credentials.userId);
-        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        var filter;
+        if (self.opts.filter) {
+            filter = self.opts.filter;
+        } else {
+            filter = new Filter(client.credentials.userId);
+            filter.setTimelineLimit(self.opts.initialSyncLimit);
+        }

        client.getOrCreateFilter(
            getFilterName(client.credentials.userId), filter
@@ -491,7 +497,7 @@ SyncApi.prototype._sync = function(syncOptions) {
        qps._cacheBuster = Date.now();
    }

-    if (self._syncConnectionLost) {
+    if (this.getSyncState() == 'ERROR' || this.getSyncState() == 'RECONNECTING') {
        // we think the connection is dead. If it comes back up, we won't know
        // about it till /sync returns. If the timeout= is high, this could
        // be a long time. Set it to 0 when doing retries so we don't have to wait
@@ -507,8 +513,6 @@ SyncApi.prototype._sync = function(syncOptions) {
    );

    this._currentSyncRequest.done(function(data) {
-        self._syncConnectionLost = false;
-
        // set the sync token NOW *before* processing the events. We do this so
        // if something barfs on an event we can skip it rather than constantly
        // polling with the same token.
@@ -540,18 +544,25 @@ SyncApi.prototype._sync = function(syncOptions) {
                self._connectionReturnedDefer.reject();
                self._connectionReturnedDefer = null;
            }
+            self._updateSyncState("STOPPED");
            return;
        }
        console.error("/sync error %s", err);
        console.error(err);

        debuglog("Starting keep-alive");
-        self._syncConnectionLost = true;
+        // Note that we do *not* mark the sync connection as
+        // lost yet: we only do this if a keepalive poke
+        // fails, since long lived HTTP connections will
+        // go away sometimes and we shouldn't treat this as
+        // erroneous. We set the state to 'reconnecting'
+        // instead, so that clients can onserve this state
+        // if they wish.
        self._startKeepAlives().done(function() {
            self._sync(syncOptions);
        });
        self._currentSyncRequest = null;
-        self._updateSyncState("ERROR", { error: err });
+        self._updateSyncState("RECONNECTING");
    });
 };

@@ -844,17 +855,21 @@ SyncApi.prototype._processSyncResponse = function(syncToken, data) {
 */
 SyncApi.prototype._startKeepAlives = function(delay) {
    if (delay === undefined) {
-        delay = 5000 + Math.floor(Math.random() * 5000);
+        delay = 2000 + Math.floor(Math.random() * 5000);
    }

    if (this._keepAliveTimer !== null) {
        clearTimeout(this._keepAliveTimer);
    }
    var self = this;
-    self._keepAliveTimer = setTimeout(
-        self._pokeKeepAlive.bind(self),
-        delay
-    );
+    if (delay > 0) {
+        self._keepAliveTimer = setTimeout(
+            self._pokeKeepAlive.bind(self),
+            delay
+        );
+    } else {
+        self._pokeKeepAlive();
+    }
    if (!this._connectionReturnedDefer) {
        this._connectionReturnedDefer = q.defer();
    }
@@ -874,9 +889,15 @@ SyncApi.prototype._pokeKeepAlive = function() {
        }
    }

-    this.client._http.requestWithPrefix(
-        undefined, "GET", "/_matrix/client/versions", undefined,
-        undefined, "", 15 * 1000
+    this.client._http.request(
+        undefined, // callback
+        "GET", "/_matrix/client/versions",
+        undefined, // queryParams
+        undefined, // data
+        {
+            prefix: '',
+            localTimeoutMs: 15 * 1000,
+        }
    ).done(function() {
        success();
    }, function(err) {
@@ -892,6 +913,13 @@ SyncApi.prototype._pokeKeepAlive = function() {
                self._pokeKeepAlive.bind(self),
                5000 + Math.floor(Math.random() * 5000)
            );
+            // A keepalive has failed, so we emit the
+            // error state (whether or not this is the
+            // first failure).
+            // Note we do this after setting the timer:
+            // this lets the unit tests advance the mock
+            // clock when the get the error.
+            self._updateSyncState("ERROR", { error: err });
        }
    });
 };
@@ -234,7 +234,7 @@ TimelineWindow.prototype.paginate = function(direction, size, makeRequest,
        // remove some events from the other end, if necessary
        var excess = this._eventCount - this._windowLimit;
        if (excess > 0) {
-            this._unpaginate(excess, direction != EventTimeline.BACKWARDS);
+            this.unpaginate(excess, direction != EventTimeline.BACKWARDS);
        }
        return q(true);
    }
@@ -287,15 +287,13 @@ TimelineWindow.prototype.paginate = function(direction, size, makeRequest,


 /**
- * Trim the window to the windowlimit
+ * Remove `delta` events from the start or end of the timeline.
 *
 * @param {number}  delta           number of events to remove from the timeline
 * @param {boolean} startOfTimeline if events should be removed from the start
 *     of the timeline.
- *
- * @private
 */
-TimelineWindow.prototype._unpaginate = function(delta, startOfTimeline) {
+TimelineWindow.prototype.unpaginate = function(delta, startOfTimeline) {
    var tl = startOfTimeline ? this._start : this._end;

    // sanity-check the delta
@@ -204,7 +204,8 @@ module.exports.isFunction = function(value) {
 * @return {boolean} True if it is an array.
 */
 module.exports.isArray = function(value) {
-    return Boolean(value && value.constructor === Array);
+    return Array.isArray ? Array.isArray(value) :
+        Boolean(value && value.constructor === Array);
 };

 /**
@@ -337,6 +338,9 @@ var deepCompare = module.exports.deepCompare = function(x, y) {
 *
 * All enumerable properties, included inherited ones, are copied.
 *
+ * This is approximately equivalent to ES6's Object.assign, except
+ * that the latter doesn't copy inherited properties.
+ *
 * @param {Object} target  The object that will receive new properties
 * @param {...Object} source  Objects from which to copy properties
 *
@@ -1,18 +1,17 @@
 {
  "name": "matrix-js-sdk",
-  "version": "0.6.1",
+  "version": "0.7.1-rc.1",
  "description": "Matrix Client-Server SDK for Javascript",
  "main": "index.js",
  "scripts": {
    "test": "istanbul cover --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" jasmine-node -- spec --verbose --junitreport --captureExceptions",
    "check": "jasmine-node spec --verbose --junitreport --captureExceptions",
    "gendoc": "jsdoc -r lib -P package.json -R README.md -d .jsdoc",
-    "build": "jshint -c .jshint lib/ && browserify browser-index.js -o dist/browser-matrix-dev.js --ignore-missing",
-    "watch": "watchify browser-index.js -o dist/browser-matrix-dev.js -v",
+    "build": "jshint -c .jshint lib/ && rimraf dist && mkdir dist && browserify --exclude olm browser-index.js -o dist/browser-matrix.js --ignore-missing && uglifyjs -c -m -o dist/browser-matrix.min.js dist/browser-matrix.js",
+    "dist": "npm run build",
+    "watch": "watchify --exclude olm browser-index.js -o dist/browser-matrix-dev.js -v",
    "lint": "jshint -c .jshint lib spec && gjslint --unix_mode --disable 0131,0211,0200,0222,0212 --max_line_length 90 -r spec/ -r lib/",
-    "release": "npm run build && mkdir -p dist/$npm_package_version && uglifyjs -c -m -o dist/$npm_package_version/browser-matrix-$npm_package_version.min.js dist/browser-matrix-dev.js && cp dist/browser-matrix-dev.js dist/$npm_package_version/browser-matrix-$npm_package_version.js",
-    "prepublish": "git rev-parse HEAD > git-revision.txt",
-    "version": "npm run release && git add dist/$npm_package_version"
+    "prepublish": "git rev-parse HEAD > git-revision.txt"
  },
  "repository": {
    "url": "https://github.com/matrix-org/matrix-js-sdk"
@@ -23,6 +22,23 @@
  "browser": "browser-index.js",
  "author": "matrix.org",
  "license": "Apache-2.0",
+  "files": [
+    "CHANGELOG.md",
+    "CONTRIBUTING.rst",
+    "LICENSE",
+    "README.md",
+    "RELEASING.md",
+    "examples",
+    "git-hooks",
+    "git-revision.txt",
+    "index.js",
+    "browser-index.js",
+    "jenkins.sh",
+    "lib",
+    "package.json",
+    "release.sh",
+    "spec"
+  ],
  "dependencies": {
    "another-json": "^0.2.0",
    "browser-request": "^0.3.3",
@@ -31,12 +47,13 @@
    "request": "^2.53.0"
  },
  "devDependencies": {
-    "watchify": "^3.2.1",
    "istanbul": "^0.3.13",
    "jasmine-node": "^1.14.5",
-    "jshint": "^2.8.0",
    "jsdoc": "^3.4.0",
-    "uglifyjs": "^2.4.10"
+    "jshint": "^2.8.0",
+    "rimraf": "^2.5.4",
+    "uglifyjs": "^2.4.10",
+    "watchify": "^3.2.1"
  },
  "optionalDependencies": {
    "olm": "https://matrix.org/packages/npm/olm/olm-1.3.0.tgz"
@@ -1,13 +1,18 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Script to perform a release of matrix-js-sdk. Performs the steps documented
 # in RELEASING.md
 #
-# Requires github-changelog-generator; to install, do
-#   pip install git+https://github.com/matrix-org/github-changelog-generator.git
+# Requires:
+#   github-changelog-generator; to install, do
+#     pip install git+https://github.com/matrix-org/github-changelog-generator.git
+#   jq; install from your distibution's package manager (https://stedolan.github.io/jq/)

 set -e

+jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
+hub --version > /dev/null || (echo "hub is required: please install it"; kill $$)
+
 USAGE="$0 [-xz] [-c changelog_file] vX.Y.Z"

 help() {
@@ -20,6 +25,13 @@ $USAGE
 EOF
 }

+ret=0
+cat package.json | jq '.dependencies[]' | grep -q '#develop' || ret=$?
+if [ "$ret" -eq 0 ]; then
+    echo "package.json contains develop dependencies. Refusing to release."
+    exit
+fi
+
 skip_changelog=
 skip_jsdoc=
 changelog_file="CHANGELOG.md"
@@ -47,11 +59,27 @@ if [ $# -ne 1 ]; then
    exit 1
 fi

+if [ -z "$skip_changelog" ]; then
+    # update_changelog doesn't have a --version flag
+    update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
+fi
+
 # ignore leading v on release
 release="${1#v}"
 tag="v${release}"
 rel_branch="release-$tag"

+prerelease=0
+# We check if this build is a prerelease by looking to
+# see if the version has a hyphen in it. Crude,
+# but semver doesn't support postreleases so anything
+# with a hyphen is a prerelease.
+echo $release | grep -q '-' && prerelease=1
+
+if [ $prerelease -eq 1 ]; then
+    echo Making a PRE-RELEASE
+fi
+
 if [ -z "$skip_changelog" ]; then
    if ! command -v update_changelog >/dev/null 2>&1; then
        echo "release.sh requires github-changelog-generator. Try:" >&2
@@ -61,9 +89,16 @@ if [ -z "$skip_changelog" ]; then
 fi

 # we might already be on the release branch, in which case, yay
-if [ $(git symbolic-ref --short HEAD) != "$rel_branch" ]; then
+# If we're on any branch starting with 'release', we don't create
+# a separate release branch (this allows us to use the same
+# release branch for releases and release candidates).
+curbranch=$(git symbolic-ref --short HEAD)
+if [[ "$curbranch" != release* ]]; then
    echo "Creating release branch"
    git checkout -b "$rel_branch"
+else
+    echo "Using current branch ($curbranch) for release"
+    rel_branch=$curbranch
 fi

 if [ -z "$skip_changelog" ]; then
@@ -81,7 +116,54 @@ set -x

 # Bump package.json, build the dist, and tag
 echo "npm version"
-npm version "$release"
+# npm version will automatically commit its modification
+# and make a release tag. We don't want it to create the tag
+# because github will do that, but we can only turn off both
+# of these behaviours, so we have to manually commit the
+# result.
+npm version --no-git-tag-version "$release"
+git commit package.json -m "$tag"
+
+# If there is a 'dist' script in the package.json,
+# run it in a separate checkout of the project, then
+# upload any files in the 'dist' directory as release
+# assets.
+# We make a completely separate checkout to be sure
+# we're using released versions of the dependencies
+# (rather than whatever we're pulling in from npm link)
+assets=''
+dodist=0
+jq -e .scripts.dist package.json 2> /dev/null || dodist=$?
+if [ $dodist -eq 0 ]; then
+    projdir=`pwd`
+    builddir=`mktemp -d 2>/dev/null || mktemp -d -t 'mytmpdir'`
+    echo "Building distribution copy in $builddir"
+    pushd "$builddir"
+    git clone "$projdir" .
+    git checkout "$rel_branch"
+    npm install
+    # We haven't tagged yet, so tell the dist script what version
+    # it's building
+    DIST_VERSION="$tag" npm run dist
+    popd
+    for i in "$builddir"/dist/*; do
+        assets="$assets -a $i"
+    done
+fi
+
+# push the release branch (github can't release from
+# a branch it doesn't have)
+git push origin "$rel_branch"
+
+hubflags=''
+if [ $prerelease -eq 1 ]; then
+    hubflags='-p'
+fi
+hub release create $hubflags $assets -m "$tag" "$tag"
+
+if [ $dodist -eq 0 ]; then
+    rm -rf "$builddir"
+fi

 if [ -z "$skip_jsdoc" ]; then
    echo "generating jsdocs"
@@ -104,8 +186,8 @@ git checkout master
 git pull
 git merge --ff-only "$rel_branch"

-# push everything to github
-git push origin master "$rel_branch" "$tag"
+# push master  and docs (if generated) to github
+git push origin master
 if [ -z "$skip_jsdoc" ]; then
    git push origin gh-pages
 fi
@@ -5,21 +5,6 @@ var HttpBackend = require("../mock-request");
 var utils = require("../../lib/utils");
 var test_utils = require("../test-utils");

-function MockStorageApi() {
-    this.data = {};
-}
-MockStorageApi.prototype = {
-    setItem: function(k, v) {
-        this.data[k] = v;
-    },
-    getItem: function(k) {
-        return this.data[k] || null;
-    },
-    removeItem: function(k) {
-        delete this.data[k];
-    }
-};
-
 var aliHttpBackend;
 var bobHttpBackend;
 var aliClient;
@@ -36,7 +21,6 @@ var aliDeviceKeys;
 var bobDeviceKeys;
 var bobDeviceCurve25519Key;
 var bobDeviceEd25519Key;
-var aliLocalStore;
 var aliStorage;
 var bobStorage;
 var aliMessages;
@@ -61,7 +45,7 @@ function expectKeyUpload(deviceId, httpBackend) {
        expect(content.one_time_keys).not.toBeDefined();
        expect(content.device_keys).toBeDefined();
        keys.device_keys = content.device_keys;
-        return {one_time_key_counts: {curve25519: 0}};
+        return {one_time_key_counts: {signed_curve25519: 0}};
    });

    httpBackend.when("POST", uploadPath).respond(200, function(path, content) {
@@ -76,7 +60,7 @@ function expectKeyUpload(deviceId, httpBackend) {
        }
        expect(count).toEqual(5);
        keys.one_time_keys = content.one_time_keys;
-        return {one_time_key_counts: {curve25519: count}};
+        return {one_time_key_counts: {signed_curve25519: count}};
    });

    return httpBackend.flush(uploadPath, 2).then(function() {
@@ -176,10 +160,11 @@ function expectAliClaimKeys() {
    expect(bobOneTimeKeys).toBeDefined();

    aliHttpBackend.when("POST", "/keys/claim").respond(200, function(path, content) {
-        expect(content.one_time_keys[bobUserId][bobDeviceId]).toEqual("curve25519");
+        var claimType = content.one_time_keys[bobUserId][bobDeviceId];
+        expect(claimType).toEqual("signed_curve25519");
        for (var keyId in bobOneTimeKeys) {
            if (bobOneTimeKeys.hasOwnProperty(keyId)) {
-                if (keyId.indexOf("curve25519:") === 0) {
+                if (keyId.indexOf(claimType + ":") === 0) {
                    break;
                }
            }
@@ -338,15 +323,15 @@ function expectSendMessageRequest(httpBackend) {

 function aliRecvMessage() {
    var message = bobMessages.shift();
-    return recvMessage(aliHttpBackend, aliClient, message);
+    return recvMessage(aliHttpBackend, aliClient, bobUserId, message);
 }

 function bobRecvMessage() {
    var message = aliMessages.shift();
-    return recvMessage(bobHttpBackend, bobClient, message);
+    return recvMessage(bobHttpBackend, bobClient, aliUserId, message);
 }

-function recvMessage(httpBackend, client, message) {
+function recvMessage(httpBackend, client, sender, message) {
    var syncData = {
        next_batch: "x",
        rooms: {
@@ -361,7 +346,8 @@ function recvMessage(httpBackend, client, message) {
                test_utils.mkEvent({
                    type: "m.room.encrypted",
                    room: roomId,
-                    content: message
+                    content: message,
+                    sender: sender,
                })
            ]
        }
@@ -459,11 +445,9 @@ describe("MatrixClient crypto", function() {
    }

    beforeEach(function() {
-        aliLocalStore = new MockStorageApi();
-        aliStorage = new sdk.WebStorageSessionStore(aliLocalStore);
-        bobStorage = new sdk.WebStorageSessionStore(new MockStorageApi());
        test_utils.beforeEach(this);

+        aliStorage = new sdk.WebStorageSessionStore(new test_utils.MockStorageApi());
        aliHttpBackend = new HttpBackend();
        aliClient = sdk.createClient({
            baseUrl: "http://alis.server",
@@ -474,6 +458,7 @@ describe("MatrixClient crypto", function() {
            request: aliHttpBackend.requestFn,
        });

+        bobStorage = new sdk.WebStorageSessionStore(new test_utils.MockStorageApi());
        bobHttpBackend = new HttpBackend();
        bobClient = sdk.createClient({
            baseUrl: "http://bobs.server",
@@ -498,6 +483,27 @@ describe("MatrixClient crypto", function() {
        bobClient.stopClient();
    });

+    it('Ali knows the difference between a new user and one with no devices',
+        function(done) {
+            aliHttpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:id': {},
+                }
+            });
+
+            var p1 = aliClient.downloadKeys(['@bob:id']);
+            var p2 = aliHttpBackend.flush('/keys/query', 1);
+
+            q.all([p1, p2]).then(function() {
+                var devices = aliStorage.getEndToEndDevicesForUser('@bob:id');
+                expect(utils.keys(devices).length).toEqual(0);
+
+                // request again: should be no more requests
+                return aliClient.downloadKeys(['@bob:id']);
+            }).nodeify(done);
+        }
+    );
+
    it("Bob uploads without one-time keys and with one-time keys", function(done) {
        q()
            .then(bobUploadsKeys)
@@ -529,6 +535,77 @@ describe("MatrixClient crypto", function() {
            .catch(test_utils.failTest).done(done);
    });

+    it("Ali gets keys with an incorrect userId", function(done) {
+        var eveUserId = "@eve:localhost";
+
+        var bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bvcxz',
+            keys: {
+                'ed25519:bvcxz': 'pYuWKMCVuaDLRTM/eWuB8OlXEb61gZhfLVJ+Y54tl0Q',
+                'curve25519:bvcxz': '7Gni0loo/nzF0nFp9847RbhElGewzwUXHPrljjBGPTQ',
+            },
+            user_id: '@eve:localhost',
+            signatures: {
+                '@eve:localhost': {
+                    'ed25519:bvcxz': 'CliUPZ7dyVPBxvhSA1d+X+LYa5b2AYdjcTwG' +
+                        '0stXcIxjaJNemQqtdgwKDtBFl3pN2I13SEijRDCf1A8bYiQMDg',
+                },
+            },
+        };
+
+        var bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+            var result = {};
+            result[bobUserId] = bobKeys;
+            return {device_keys: result};
+        });
+
+        q.all(
+            aliClient.downloadKeys([bobUserId, eveUserId]),
+            aliHttpBackend.flush("/keys/query", 1)
+        ).then(function() {
+            // should get an empty list
+            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+            expect(aliClient.listDeviceKeys(eveUserId)).toEqual([]);
+        }).catch(test_utils.failTest).done(done);
+    });
+
+    it("Ali gets keys with an incorrect deviceId", function(done) {
+        var bobDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'bad_device',
+            keys: {
+                'ed25519:bad_device': 'e8XlY5V8x2yJcwa5xpSzeC/QVOrU+D5qBgyTK0ko+f0',
+                'curve25519:bad_device': 'YxuuLG/4L5xGeP8XPl5h0d7DzyYVcof7J7do+OXz0xc',
+            },
+            user_id: '@bob:localhost',
+            signatures: {
+                '@bob:localhost': {
+                    'ed25519:bad_device': 'fEFTq67RaSoIEVBJ8DtmRovbwUBKJ0A' +
+                        'me9m9PDzM9azPUwZ38Xvf6vv1A7W1PSafH4z3Y2ORIyEnZgHaNby3CQ',
+                },
+            },
+        };
+
+        var bobKeys = {};
+        bobKeys[bobDeviceId] = bobDeviceKeys;
+        aliHttpBackend.when("POST", "/keys/query").respond(200, function(path, content) {
+            var result = {};
+            result[bobUserId] = bobKeys;
+            return {device_keys: result};
+        });
+
+        q.all(
+            aliClient.downloadKeys([bobUserId]),
+            aliHttpBackend.flush("/keys/query", 1)
+        ).then(function() {
+            // should get an empty list
+            expect(aliClient.listDeviceKeys(bobUserId)).toEqual([]);
+        }).catch(test_utils.failTest).done(done);
+    });
+
    it("Ali enables encryption", function(done) {
        q()
            .then(bobUploadsKeys)
@@ -557,6 +634,63 @@ describe("MatrixClient crypto", function() {
            .catch(test_utils.failTest).done(done);
    });

+    it("Bob receives a message with a bogus sender", function(done) {
+        q()
+            .then(bobUploadsKeys)
+            .then(aliStartClient)
+            .then(aliEnablesEncryption)
+            .then(aliSendsFirstMessage)
+            .then(bobStartClient)
+            .then(function() {
+                var message = aliMessages.shift();
+                var syncData = {
+                    next_batch: "x",
+                    rooms: {
+                        join: {
+
+                        }
+                    }
+                };
+                syncData.rooms.join[roomId] = {
+                    timeline: {
+                        events: [
+                            test_utils.mkEvent({
+                                type: "m.room.encrypted",
+                                room: roomId,
+                                content: message,
+                                sender: "@bogus:sender",
+                            })
+                        ]
+                    }
+                };
+                bobHttpBackend.when("GET", "/sync").respond(200, syncData);
+
+                var deferred = q.defer();
+                var onEvent = function(event) {
+                    console.log(bobClient.credentials.userId + " received event",
+                                event);
+
+                    // ignore the m.room.member events
+                    if (event.getType() == "m.room.member") {
+                        return;
+                    }
+
+                    expect(event.getType()).toEqual("m.room.message");
+                    expect(event.getContent().msgtype).toEqual("m.bad.encrypted");
+                    expect(event.isEncrypted()).toBeTruthy();
+
+                    bobClient.removeListener("event", onEvent);
+                    deferred.resolve();
+                };
+
+                bobClient.on("event", onEvent);
+
+                bobHttpBackend.flush();
+                return deferred.promise;
+            })
+            .catch(test_utils.failTest).done(done);
+    });
+
    it("Ali blocks Bob's device", function(done) {
        q()
            .then(bobUploadsKeys)
@@ -605,4 +739,31 @@ describe("MatrixClient crypto", function() {
            }).then(aliRecvMessage)
            .catch(test_utils.failTest).done(done);
    });
+
+
+    it("Ali does a key query when she gets a new_device event", function(done) {
+        q()
+            .then(bobUploadsKeys)
+            .then(aliStartClient)
+            .then(function() {
+                var syncData = {
+                    next_batch: '2',
+                    to_device: {
+                        events: [
+                            test_utils.mkEvent({
+                                content: {
+                                    device_id: 'TEST_DEVICE',
+                                    rooms: [],
+                                },
+                                sender: bobUserId,
+                                type: 'm.new_device',
+                            }),
+                        ],
+                    },
+                };
+                aliHttpBackend.when('GET', '/sync').respond(200, syncData);
+                return aliHttpBackend.flush('/sync', 1);
+            }).then(expectAliQueryKeys)
+            .nodeify(done);
+    });
 });
@@ -37,6 +37,117 @@ describe("MatrixClient", function() {
        httpBackend.verifyNoOutstandingExpectation();
    });

+    describe("uploadContent", function() {
+        var buf = new Buffer('hello world');
+        it("should upload the file", function(done) {
+            httpBackend.when(
+                "POST", "/_matrix/media/v1/upload"
+            ).check(function(req) {
+                expect(req.data).toEqual(buf);
+                expect(req.queryParams.filename).toEqual("hi.txt");
+                expect(req.queryParams.access_token).toEqual(accessToken);
+                expect(req.headers["Content-Type"]).toEqual("text/plain");
+                expect(req.opts.json).toBeFalsy();
+                expect(req.opts.timeout).toBe(undefined);
+            }).respond(200, "content");
+
+            var prom = client.uploadContent({
+                stream: buf,
+                name: "hi.txt",
+                type: "text/plain",
+            });
+
+            expect(prom).toBeDefined();
+
+            var uploads = client.getCurrentUploads();
+            expect(uploads.length).toEqual(1);
+            expect(uploads[0].promise).toBe(prom);
+            expect(uploads[0].loaded).toEqual(0);
+
+            prom.then(function(response) {
+                // for backwards compatibility, we return the raw JSON
+                expect(response).toEqual("content");
+
+                var uploads = client.getCurrentUploads();
+                expect(uploads.length).toEqual(0);
+            }).catch(utils.failTest).done(done);
+
+            httpBackend.flush();
+        });
+
+        it("should parse the response if rawResponse=false", function(done) {
+            httpBackend.when(
+                "POST", "/_matrix/media/v1/upload"
+            ).check(function(req) {
+                expect(req.opts.json).toBeFalsy();
+            }).respond(200, JSON.stringify({ "content_uri": "uri" }));
+
+            client.uploadContent({
+                stream: buf,
+                name: "hi.txt",
+                type: "text/plain",
+            }, {
+                rawResponse: false,
+            }).then(function(response) {
+                expect(response.content_uri).toEqual("uri");
+            }).catch(utils.failTest).done(done);
+
+            httpBackend.flush();
+        });
+
+        it("should parse errors into a MatrixError", function(done) {
+            // opts.json is false, so request returns unparsed json.
+            httpBackend.when(
+                "POST", "/_matrix/media/v1/upload"
+            ).check(function(req) {
+                expect(req.data).toEqual(buf);
+                expect(req.opts.json).toBeFalsy();
+            }).respond(400, JSON.stringify({
+                "errcode": "M_SNAFU",
+                "error": "broken",
+            }));
+
+            client.uploadContent({
+                stream: buf,
+                name: "hi.txt",
+                type: "text/plain",
+            }).then(function(response) {
+                throw Error("request not failed");
+            }, function(error) {
+                expect(error.httpStatus).toEqual(400);
+                expect(error.errcode).toEqual("M_SNAFU");
+                expect(error.message).toEqual("broken");
+            }).catch(utils.failTest).done(done);
+
+            httpBackend.flush();
+        });
+
+        it("should return a promise which can be cancelled", function(done) {
+            var prom = client.uploadContent({
+                stream: buf,
+                name: "hi.txt",
+                type: "text/plain",
+            });
+
+            var uploads = client.getCurrentUploads();
+            expect(uploads.length).toEqual(1);
+            expect(uploads[0].promise).toBe(prom);
+            expect(uploads[0].loaded).toEqual(0);
+
+            prom.then(function(response) {
+                throw Error("request not aborted");
+            }, function(error) {
+                expect(error).toEqual("aborted");
+
+                var uploads = client.getCurrentUploads();
+                expect(uploads.length).toEqual(0);
+            }).catch(utils.failTest).done(done);
+
+            var r = client.cancelUpload(prom);
+            expect(r).toBe(true);
+        });
+    });
+
    describe("joinRoom", function() {
        it("should no-op if you've already joined a room", function() {
            var roomId = "!foo:bar";
@@ -184,58 +295,65 @@ describe("MatrixClient", function() {

    describe("downloadKeys", function() {
        it("should do an HTTP request and then store the keys", function(done) {
-            var ed25519key = "wV5E3EUSHpHuoZLljNzojlabjGdXT3Mz7rugG9zgbkI";
+            var ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
+            // ed25519key = client.getDeviceEd25519Key();
            var borisKeys = {
                dev1: {
-                    algorithms: ["1"], keys: { "ed25519:dev1": ed25519key },
+                    algorithms: ["1"],
+                    device_id: "dev1",
+                    keys: { "ed25519:dev1": ed25519key },
                    signatures: {
                        boris: {
-                           "ed25519:dev1":
-                                "u99n8WZ61G//K6eVgYc+RDLVapmjttxqhjNucIFGEIJ" +
-                                "oA4TUY8FmiGv3zl0EA71zrvPDfnFL5XLNsdc55NGbDg"
+                            "ed25519:dev1":
+                                "RAhmbNDq1efK3hCpBzZDsKoGSsrHUxb25NW5/WbEV9R" +
+                                "JVwLdP032mg5QsKt/pBDUGtggBcnk43n3nBWlA88WAw"
                        }
                    },
                    unsigned: { "abc": "def" },
+                    user_id: "boris",
                }
            };
            var chazKeys = {
                dev2: {
-                    algorithms: ["2"], keys: { "ed25519:dev2": ed25519key },
+                    algorithms: ["2"],
+                    device_id: "dev2",
+                    keys: { "ed25519:dev2": ed25519key },
                    signatures: {
                        chaz: {
                           "ed25519:dev2":
-                                "8eaeXUWy9AQzjaNVOjVLs4FQk+cgobkNS811EjZBCMA" +
-                                "apd8aPOfE26E13nFFOCLC1V6fOH5wVo61hxGR/j4PBA"
-                        }
-                    },
-                    unsigned: { "ghi": "def" },
-                }
-            };
-            var daveKeys = {
-                dev3: {
-                    algorithms: ["3"], keys: { "ed25519:dev2": ed25519key },
-                    signatures: {
-                        dave: {
-                           "ed25519:dev2":
-                                "8eaeXUWy9AQzjaNVOjVLs4FQk+cgobkNS811EjZBCMA" +
-                                "apd8aPOfE26E13nFFOCLC1V6fOH5wVo61hxGR/j4PBA"
+                                "FwslH/Q7EYSb7swDJbNB5PSzcbEO1xRRBF1riuijqvL" +
+                                "EkrK9/XVN8jl4h7thGuRITQ01siBQnNmMK9t45QfcCQ"
                        }
                    },
                    unsigned: { "ghi": "def" },
+                    user_id: "chaz",
                }
            };

+            /*
+            function sign(o) {
+                var anotherjson = require('another-json');
+                var b = JSON.parse(JSON.stringify(o));
+                delete(b.signatures);
+                delete(b.unsigned);
+                return client._crypto._olmDevice.sign(anotherjson.stringify(b));
+            };
+
+            console.log("Ed25519: " + ed25519key);
+            console.log("boris:", sign(borisKeys.dev1));
+            console.log("chaz:", sign(chazKeys.dev2));
+            */
+
            httpBackend.when("POST", "/keys/query").check(function(req) {
-                expect(req.data).toEqual({device_keys: {boris: {}, chaz: {}, dave: {}}});
+                expect(req.data).toEqual({device_keys: {boris: {}, chaz: {}}});
            }).respond(200, {
                device_keys: {
                    boris: borisKeys,
                    chaz: chazKeys,
-                    dave: daveKeys,
                },
            });

-            client.downloadKeys(["boris", "chaz", "dave"]).then(function(res) {
+            client.downloadKeys(["boris", "chaz"]).then(function(res) {
                assertObjectContains(res.boris.dev1, {
                    verified: 0, // DeviceVerification.UNVERIFIED
                    keys: { "ed25519:dev1": ed25519key },
@@ -249,25 +367,24 @@ describe("MatrixClient", function() {
                    algorithms: ["2"],
                    unsigned: { "ghi": "def" },
                });
-
-                // dave's key fails validation.
-                expect(res.dave).toEqual({});
            }).catch(utils.failTest).done(done);

            httpBackend.flush();
        });
+    });

-        it("should return a rejected promise if the request fails", function(done) {
-            httpBackend.when("POST", "/keys/query").respond(400);
+    describe("deleteDevice", function() {
+        var auth = {a: 1};
+        it("should pass through an auth dict", function(done) {
+            httpBackend.when(
+                "DELETE", "/_matrix/client/unstable/devices/my_device"
+            ).check(function(req) {
+                expect(req.data).toEqual({auth: auth});
+            }).respond(200);

-            var exceptionThrown;
-            client.downloadKeys(["bottom"]).then(function() {
-                fail("download didn't fail");
-            }, function(err) {
-                exceptionThrown = err;
-            }).then(function() {
-                expect(exceptionThrown).toBeTruthy();
-            }).catch(utils.failTest).done(done);
+            client.deleteDevice(
+                "my_device", auth
+            ).catch(utils.failTest).done(done);

            httpBackend.flush();
        });
@@ -0,0 +1,617 @@
+/*
+Copyright 2016 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+"use strict";
+
+
+try {
+    var Olm = require('olm');
+} catch (e) {}
+
+var anotherjson = require('another-json');
+var q = require('q');
+
+var sdk = require('../..');
+var utils = require('../../lib/utils');
+var test_utils = require('../test-utils');
+var MockHttpBackend = require('../mock-request');
+
+/**
+ * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
+ *
+ * @constructor
+ * @param {string} userId
+ * @param {string} deviceId
+ * @param {string} accessToken
+ */
+function TestClient(userId, deviceId, accessToken) {
+    this.userId = userId;
+    this.deviceId = deviceId;
+
+    this.storage = new sdk.WebStorageSessionStore(new test_utils.MockStorageApi());
+    this.httpBackend = new MockHttpBackend();
+    this.client = sdk.createClient({
+        baseUrl: "http://test.server",
+        userId: userId,
+        accessToken: accessToken,
+        deviceId: deviceId,
+        sessionStore: this.storage,
+        request: this.httpBackend.requestFn,
+    });
+
+    this.deviceKeys = null;
+    this.oneTimeKeys = [];
+}
+
+/**
+ * start the client, and wait for it to initialise.
+ *
+ * @return {Promise}
+ */
+TestClient.prototype.start = function() {
+    var self = this;
+    this.httpBackend.when("GET", "/pushrules").respond(200, {});
+    this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
+        expect(content.one_time_keys).not.toBeDefined();
+        expect(content.device_keys).toBeDefined();
+        self.deviceKeys = content.device_keys;
+        return {one_time_key_counts: {signed_curve25519: 0}};
+    });
+    this.httpBackend.when("POST", "/keys/upload").respond(200, function(path, content) {
+        expect(content.device_keys).not.toBeDefined();
+        expect(content.one_time_keys).toBeDefined();
+        expect(content.one_time_keys).not.toEqual({});
+        self.oneTimeKeys = content.one_time_keys;
+        return {one_time_key_counts: {
+            signed_curve25519: utils.keys(self.oneTimeKeys).length
+        }};
+    });
+
+    this.client.startClient();
+
+    return this.httpBackend.flush();
+};
+
+/**
+ * stop the client
+ */
+TestClient.prototype.stop = function() {
+    this.client.stopClient();
+};
+
+/**
+ * get the uploaded curve25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getDeviceKey = function() {
+    var key_id = 'curve25519:' + this.deviceId;
+    return this.deviceKeys.keys[key_id];
+};
+
+
+/**
+ * get the uploaded ed25519 device key
+ *
+ * @return {string} base64 device key
+ */
+TestClient.prototype.getSigningKey = function() {
+    var key_id = 'ed25519:' + this.deviceId;
+    return this.deviceKeys.keys[key_id];
+};
+
+/**
+ * start an Olm session with a given recipient
+ *
+ * @param {Olm.Account} olmAccount
+ * @param {TestClient} recipientTestClient
+ * @return {Olm.Session}
+ */
+function createOlmSession(olmAccount, recipientTestClient) {
+    var otk_id = utils.keys(recipientTestClient.oneTimeKeys)[0];
+    var otk = recipientTestClient.oneTimeKeys[otk_id];
+
+    var session = new Olm.Session();
+    session.create_outbound(
+        olmAccount, recipientTestClient.getDeviceKey(), otk.key
+    );
+    return session;
+}
+
+/**
+ * encrypt an event with olm
+ *
+ * @param {object} opts
+ * @param {string=} opts.sender
+ * @param {string} opts.senderKey
+ * @param {Olm.Session} opts.p2pSession
+ * @param {TestClient} opts.recipient
+ * @param {object=} opts.plaincontent
+ * @param {string=} opts.plaintype
+ *
+ * @return {object} event
+ */
+function encryptOlmEvent(opts) {
+    expect(opts.senderKey).toBeDefined();
+    expect(opts.p2pSession).toBeDefined();
+    expect(opts.recipient).toBeDefined();
+
+    var plaintext = {
+        content: opts.plaincontent || {},
+        recipient: opts.recipient.userId,
+        recipient_keys: {
+            ed25519: opts.recipient.getSigningKey(),
+        },
+        sender: opts.sender || '@bob:xyz',
+        type: opts.plaintype || 'm.test',
+    };
+
+    var event = {
+        content: {
+            algorithm: 'm.olm.v1.curve25519-aes-sha2',
+            ciphertext: {},
+            sender_key: opts.senderKey,
+        },
+        sender: opts.sender || '@bob:xyz',
+        type: 'm.room.encrypted',
+    };
+    event.content.ciphertext[opts.recipient.getDeviceKey()] =
+        opts.p2pSession.encrypt(JSON.stringify(plaintext));
+    return event;
+}
+
+/**
+ * encrypt an event with megolm
+ *
+ * @param {object} opts
+ * @param {string} opts.senderKey
+ * @param {Olm.OutboundGroupSession} opts.groupSession
+ * @param {object=} opts.plaintext
+ * @param {string=} opts.room_id
+ *
+ * @return {object} event
+ */
+function encryptMegolmEvent(opts) {
+    expect(opts.senderKey).toBeDefined();
+    expect(opts.groupSession).toBeDefined();
+
+    var plaintext = opts.plaintext || {};
+    if (!plaintext.content) {
+        plaintext.content = {
+            body: '42',
+            msgtype: "m.text",
+        };
+    }
+    if (!plaintext.type) {
+        plaintext.type = "m.room.message";
+    }
+    if (!plaintext.room_id) {
+        expect(opts.room_id).toBeDefined();
+        plaintext.room_id = opts.room_id;
+    }
+
+    return {
+        content: {
+            algorithm: "m.megolm.v1.aes-sha2",
+            ciphertext: opts.groupSession.encrypt(JSON.stringify(plaintext)),
+            device_id: "testDevice",
+            sender_key: opts.senderKey,
+            session_id: opts.groupSession.session_id(),
+        },
+        type: "m.room.encrypted",
+    };
+}
+
+/**
+ * build an encrypted room_key event to share a group session
+ *
+ * @param {object} opts
+ * @param {string} opts.senderKey
+ * @param {TestClient} opts.recipient
+ * @param {Olm.Session} opts.p2pSession
+ * @param {Olm.OutboundGroupSession} opts.groupSession
+ * @param {string=} opts.room_id
+ *
+ * @return {object} event
+ */
+function encryptGroupSessionKey(opts) {
+    return encryptOlmEvent({
+        senderKey: opts.senderKey,
+        recipient: opts.recipient,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            algorithm: 'm.megolm.v1.aes-sha2',
+            room_id: opts.room_id,
+            session_id: opts.groupSession.session_id(),
+            session_key: opts.groupSession.session_key(),
+        },
+        plaintype: 'm.room_key',
+    });
+}
+
+describe("megolm", function() {
+    if (!sdk.CRYPTO_ENABLED) {
+        return;
+    }
+
+    var ROOM_ID = "!room:id";
+
+    var testOlmAccount;
+    var testSenderKey;
+    var aliceTestClient;
+    var testDeviceKeys;
+
+    beforeEach(function(done) {
+        test_utils.beforeEach(this);
+
+        aliceTestClient = new TestClient(
+            "@alice:localhost", "xzcvb", "akjgkrgjs"
+        );
+
+        testOlmAccount = new Olm.Account();
+        testOlmAccount.create();
+        var testE2eKeys = JSON.parse(testOlmAccount.identity_keys());
+        testSenderKey = testE2eKeys.curve25519;
+
+        testDeviceKeys = {
+            algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
+            device_id: 'DEVICE_ID',
+            keys: {
+                'curve25519:DEVICE_ID': testE2eKeys.curve25519,
+                'ed25519:DEVICE_ID': testE2eKeys.ed25519,
+            },
+            user_id: '@bob:xyz',
+        };
+        var j = anotherjson.stringify(testDeviceKeys);
+        var sig = testOlmAccount.sign(j);
+        testDeviceKeys.signatures = {
+            '@bob:xyz': {
+                'ed25519:DEVICE_ID': sig,
+            },
+        };
+
+        return aliceTestClient.start().nodeify(done);
+    });
+
+    afterEach(function() {
+        aliceTestClient.stop();
+    });
+
+    it("Alice receives a megolm message", function(done) {
+        var p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+
+        var groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+
+        // make the room_key event
+        var roomKeyEncrypted = encryptGroupSessionKey({
+            senderKey: testSenderKey,
+            recipient: aliceTestClient,
+            p2pSession: p2pSession,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+
+        // encrypt a message with the group session
+        var messageEncrypted = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+
+        // Alice gets both the events in a single sync
+        var syncResponse = {
+            next_batch: 1,
+            to_device: {
+                events: [roomKeyEncrypted],
+            },
+            rooms: {
+                join: {},
+            },
+        };
+        syncResponse.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [messageEncrypted],
+            },
+        };
+        aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse);
+        return aliceTestClient.httpBackend.flush("/sync", 1).then(function() {
+            var room = aliceTestClient.client.getRoom(ROOM_ID);
+            var event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().body).toEqual('42');
+        }).nodeify(done);
+    });
+
+    it("Alice gets a second room_key message", function(done) {
+        var p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+
+        var groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+
+        // make the room_key event
+        var roomKeyEncrypted1 = encryptGroupSessionKey({
+            senderKey: testSenderKey,
+            recipient: aliceTestClient,
+            p2pSession: p2pSession,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+
+        // encrypt a message with the group session
+        var messageEncrypted = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+
+        // make a second room_key event now that we have advanced the group
+        // session.
+        var roomKeyEncrypted2 = encryptGroupSessionKey({
+            senderKey: testSenderKey,
+            recipient: aliceTestClient,
+            p2pSession: p2pSession,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+
+        // on the first sync, send the best room key
+        aliceTestClient.httpBackend.when("GET", "/sync").respond(200, {
+            next_batch: 1,
+            to_device: {
+                events: [roomKeyEncrypted1],
+            },
+        });
+
+        // on the second sync, send the advanced room key, along with the
+        // message.  This simulates the situation where Alice has been sent a
+        // later copy of the room key and is reloading the client.
+        var syncResponse2 = {
+            next_batch: 2,
+            to_device: {
+                events: [roomKeyEncrypted2],
+            },
+            rooms: {
+                join: {},
+            },
+        };
+        syncResponse2.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [messageEncrypted],
+            },
+        };
+        aliceTestClient.httpBackend.when("GET", "/sync").respond(200, syncResponse2);
+
+        return aliceTestClient.httpBackend.flush("/sync", 2).then(function() {
+            var room = aliceTestClient.client.getRoom(ROOM_ID);
+            var event = room.getLiveTimeline().getEvents()[0];
+            expect(event.getContent().body).toEqual('42');
+        }).nodeify(done);
+    });
+
+    it('Alice sends a megolm message', function(done) {
+        // establish an olm session with alice
+        var p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+
+        var olmEvent = encryptOlmEvent({
+            senderKey: testSenderKey,
+            recipient: aliceTestClient,
+            p2pSession: p2pSession,
+        });
+
+        var syncResponse = {
+            next_batch: 1,
+            to_device: {
+                events: [olmEvent],
+            },
+            rooms: {
+                join: {},
+            },
+        };
+        syncResponse.rooms.join[ROOM_ID] = {
+            state: {
+                events: [
+                    test_utils.mkEvent({
+                        type: 'm.room.encryption',
+                        skey: '',
+                        content: {
+                            algorithm: 'm.megolm.v1.aes-sha2',
+                        },
+                    }),
+                    test_utils.mkMembership({
+                        mship: 'join',
+                        sender: '@bob:xyz',
+                    }),
+                ],
+            },
+        };
+        aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+        var inboundGroupSession;
+
+        return aliceTestClient.httpBackend.flush('/sync', 1).then(function() {
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {
+                        'DEVICE_ID': testDeviceKeys,
+                    },
+                }
+            });
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/m.room.encrypted/'
+            ).respond(200, function(path, content) {
+                var m = content.messages['@bob:xyz'].DEVICE_ID;
+                var ct = m.ciphertext[testSenderKey];
+                var decrypted = JSON.parse(p2pSession.decrypt(ct.type, ct.body));
+
+                expect(decrypted.type).toEqual('m.room_key');
+                inboundGroupSession = new Olm.InboundGroupSession();
+                inboundGroupSession.create(decrypted.content.session_key);
+                return {};
+            });
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/'
+            ).respond(200, function(path, content) {
+                var ct = content.ciphertext;
+                var decrypted = JSON.parse(inboundGroupSession.decrypt(ct));
+
+                console.log('Decrypted received megolm message', decrypted);
+                expect(decrypted.type).toEqual('m.room.message');
+                expect(decrypted.content.body).toEqual('test');
+
+                return {
+                    event_id: '$event_id',
+                };
+            });
+
+            return q.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                aliceTestClient.httpBackend.flush(),
+            ]);
+        }).nodeify(done);
+    });
+
+    it("Alice shouldn't do a second /query for non-e2e-capable devices", function(done) {
+        var syncResponse = {
+            next_batch: 1,
+            rooms: {
+                join: {},
+            },
+        };
+        syncResponse.rooms.join[ROOM_ID] = {
+            state: {
+                events: [
+                    test_utils.mkEvent({
+                        type: 'm.room.encryption',
+                        skey: '',
+                        content: {
+                            algorithm: 'm.megolm.v1.aes-sha2',
+                        },
+                    }),
+                    test_utils.mkMembership({
+                        mship: 'join',
+                        sender: '@bob:xyz',
+                    }),
+                ],
+            },
+        };
+        aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+        return aliceTestClient.httpBackend.flush('/sync', 1).then(function() {
+            console.log("Forcing alice to download our device keys");
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {},
+                }
+            });
+
+            return q.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log("Telling alice to send a megolm message");
+
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/'
+            ).respond(200, {
+                    event_id: '$event_id',
+            });
+
+            return q.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                aliceTestClient.httpBackend.flush(),
+            ]);
+        }).nodeify(done);
+    });
+
+
+    it("We shouldn't attempt to send to blocked devices", function(done) {
+        // establish an olm session with alice
+        var p2pSession = createOlmSession(testOlmAccount, aliceTestClient);
+
+        var olmEvent = encryptOlmEvent({
+            senderKey: testSenderKey,
+            recipient: aliceTestClient,
+            p2pSession: p2pSession,
+        });
+
+        var syncResponse = {
+            next_batch: 1,
+            to_device: {
+                events: [olmEvent],
+            },
+            rooms: {
+                join: {},
+            },
+        };
+
+        syncResponse.rooms.join[ROOM_ID] = {
+            state: {
+                events: [
+                    test_utils.mkEvent({
+                        type: 'm.room.encryption',
+                        skey: '',
+                        content: {
+                            algorithm: 'm.megolm.v1.aes-sha2',
+                        },
+                    }),
+                    test_utils.mkMembership({
+                        mship: 'join',
+                        sender: '@bob:xyz',
+                    }),
+                ],
+            },
+        };
+        aliceTestClient.httpBackend.when('GET', '/sync').respond(200, syncResponse);
+
+        return aliceTestClient.httpBackend.flush('/sync', 1).then(function() {
+            console.log('Forcing alice to download our device keys');
+
+            aliceTestClient.httpBackend.when('POST', '/keys/query').respond(200, {
+                device_keys: {
+                    '@bob:xyz': {
+                        'DEVICE_ID': testDeviceKeys,
+                    },
+                }
+            });
+
+            return q.all([
+                aliceTestClient.client.downloadKeys(['@bob:xyz']),
+                aliceTestClient.httpBackend.flush('/keys/query', 1),
+            ]);
+        }).then(function() {
+            console.log('Telling alice to block our device');
+            aliceTestClient.client.setDeviceBlocked('@bob:xyz', 'DEVICE_ID');
+
+            console.log('Telling alice to send a megolm message');
+            aliceTestClient.httpBackend.when(
+                'PUT', '/send/'
+            ).respond(200, {
+                event_id: '$event_id',
+            });
+
+            return q.all([
+                aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
+                aliceTestClient.httpBackend.flush(),
+            ]);
+        }).nodeify(done);
+    });
+
+});
@@ -11,18 +11,17 @@ function HttpBackend() {
    var self = this;
    // the request function dependency that the SDK needs.
    this.requestFn = function(opts, callback) {
-        var realReq = new Request(opts.method, opts.uri, opts.body, opts.qs);
-        realReq.callback = callback;
-        console.log("HTTP backend received request: %s %s", opts.method, opts.uri);
-        self.requests.push(realReq);
+        var req = new Request(opts, callback);
+        console.log("HTTP backend received request: %s", req);
+        self.requests.push(req);

        var abort = function() {
-            var idx = self.requests.indexOf(realReq);
+            var idx = self.requests.indexOf(req);
            if (idx >= 0) {
                console.log("Aborting HTTP request: %s %s", opts.method,
                            opts.uri);
                self.requests.splice(idx, 1);
-                realReq.callback("aborted");
+                req.callback("aborted");
            }
        };

@@ -161,22 +160,32 @@ HttpBackend.prototype = {
     * @return {Request} An expected request.
     */
    when: function(method, path, data) {
-        var pendingReq = new Request(method, path, data);
+        var pendingReq = new ExpectedRequest(method, path, data);
        this.expectedRequests.push(pendingReq);
        return pendingReq;
    }
 };

-function Request(method, path, data, queryParams) {
+/**
+ * Represents the expectation of a request.
+ *
+ * <p>Includes the conditions to be matched against, the checks to be made,
+ * and the response to be returned.
+ *
+ * @constructor
+ * @param {string} method
+ * @param {string} path
+ * @param {object?} data
+ */
+function ExpectedRequest(method, path, data) {
    this.method = method;
    this.path = path;
    this.data = data;
-    this.queryParams = queryParams;
-    this.callback = null;
    this.response = null;
    this.checks = [];
 }
-Request.prototype = {
+
+ExpectedRequest.prototype = {
    /**
     * Execute a check when this request has been satisfied.
     * @param {Function} fn The function to execute.
@@ -221,6 +230,44 @@ Request.prototype = {
    }
 };

+/**
+ * Represents a request made by the app.
+ *
+ * @constructor
+ * @param {object} opts opts passed to request()
+ * @param {function} callback
+ */
+function Request(opts, callback) {
+    this.opts = opts;
+    this.callback = callback;
+
+    Object.defineProperty(this, 'method', {
+        get: function() { return opts.method; }
+    });
+
+    Object.defineProperty(this, 'path', {
+        get: function() { return opts.uri; }
+    });
+
+    Object.defineProperty(this, 'data', {
+        get: function() { return opts.body; }
+    });
+
+    Object.defineProperty(this, 'queryParams', {
+        get: function() { return opts.qs; }
+    });
+
+    Object.defineProperty(this, 'headers', {
+        get: function() { return opts.headers || {}; }
+    });
+}
+
+Request.prototype = {
+    toString: function() {
+        return this.method + " " + this.path;
+    },
+};
+
 /**
 * The HttpBackend class.
 */
@@ -65,7 +65,7 @@ module.exports.mkEvent = function(opts) {
        content: opts.content,
        event_id: "$" + Math.random() + "-" + Math.random()
    };
-    if (opts.skey) {
+    if (opts.skey !== undefined) {
        event.state_key = opts.skey;
    }
    else if (["m.room.name", "m.room.topic", "m.room.create", "m.room.join_rules",
@@ -159,7 +159,16 @@ module.exports.mkMessage = function(opts) {
 * <p>This is useful for use with integration tests which use asyncronous
 * methods: it can be added as a 'catch' handler in a promise chain.
 *
- * @param {Error} error   exception to be reported
+ * @param {Error} err   exception to be reported
+ *
+ * @deprecated
+ * It turns out there are easier ways of doing this. Just use nodeify():
+ *
+ * it("should not throw", function(done) {
+ *    asynchronousMethod().then(function() {
+ *       // some tests
+ *    }).nodeify(done);
+ * });
 *
 * @example
 * it("should not throw", function(done) {
@@ -168,6 +177,27 @@ module.exports.mkMessage = function(opts) {
 *    }).catch(utils.failTest).done(done);
 * });
 */
-module.exports.failTest = function(error) {
-    expect(error.stack).toBe(null);
+module.exports.failTest = function(err) {
+    expect(true).toBe(false, "Testfunc threw: " + err.stack);
+};
+
+
+/**
+ * A mock implementation of webstorage
+ *
+ * @constructor
+ */
+module.exports.MockStorageApi = function() {
+    this.data = {};
+};
+module.exports.MockStorageApi.prototype = {
+    setItem: function(k, v) {
+        this.data[k] = v;
+    },
+    getItem: function(k) {
+        return this.data[k] || null;
+    },
+    removeItem: function(k) {
+        delete this.data[k];
+    }
 };
@@ -0,0 +1,141 @@
+/*
+Copyright 2016 OpenMarket Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+"use strict";
+
+var q = require("q");
+var sdk = require("../..");
+var utils = require("../test-utils");
+
+var InteractiveAuth = sdk.InteractiveAuth;
+var MatrixError = sdk.MatrixError;
+
+describe("InteractiveAuth", function() {
+    beforeEach(function() {
+        utils.beforeEach(this);
+    });
+
+    it("should start an auth stage and complete it", function(done) {
+        var doRequest = jasmine.createSpy('doRequest');
+        var startAuthStage = jasmine.createSpy('startAuthStage');
+
+        var ia = new InteractiveAuth({
+            doRequest: doRequest,
+            startAuthStage: startAuthStage,
+            authData: {
+                session: "sessionId",
+                flows: [
+                    { stages: ["logintype"] },
+                ],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            },
+        });
+
+        expect(ia.getSessionId()).toEqual("sessionId");
+        expect(ia.getStageParams("logintype")).toEqual({
+            param: "aa",
+        });
+
+        // first we expect a call here
+        startAuthStage.andCallFake(function(stage) {
+            expect(stage).toEqual("logintype");
+            ia.submitAuthDict({
+                type: "logintype",
+                foo: "bar",
+            });
+        });
+
+        // .. which should trigger a call here
+        var requestRes = {"a": "b"};
+        doRequest.andCallFake(function(authData) {
+            expect(authData).toEqual({
+                session: "sessionId",
+                type: "logintype",
+                foo: "bar",
+            });
+            return q(requestRes);
+        });
+
+        ia.attemptAuth().then(function(res) {
+            expect(res).toBe(requestRes);
+            expect(doRequest.calls.length).toEqual(1);
+            expect(startAuthStage.calls.length).toEqual(1);
+        }).catch(utils.failTest).done(done);
+    });
+
+    it("should make a request if no authdata is provided", function(done) {
+        var doRequest = jasmine.createSpy('doRequest');
+        var startAuthStage = jasmine.createSpy('startAuthStage');
+
+        var ia = new InteractiveAuth({
+            doRequest: doRequest,
+            startAuthStage: startAuthStage,
+        });
+
+        expect(ia.getSessionId()).toBe(undefined);
+        expect(ia.getStageParams("logintype")).toBe(undefined);
+
+        // first we expect a call to doRequest
+        doRequest.andCallFake(function(authData) {
+            console.log("request1", authData);
+            expect(authData).toBe(null);
+            var err = new MatrixError({
+                session: "sessionId",
+                flows: [
+                    { stages: ["logintype"] },
+                ],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        // .. which should be followed by a call to startAuthStage
+        var requestRes = {"a": "b"};
+        startAuthStage.andCallFake(function(stage) {
+            expect(stage).toEqual("logintype");
+            expect(ia.getSessionId()).toEqual("sessionId");
+            expect(ia.getStageParams("logintype")).toEqual({
+                param: "aa",
+            });
+
+            // submitAuthDict should trigger another call to doRequest
+            doRequest.andCallFake(function(authData) {
+                console.log("request2", authData);
+                expect(authData).toEqual({
+                    session: "sessionId",
+                    type: "logintype",
+                    foo: "bar",
+                });
+                return q(requestRes);
+            });
+
+            ia.submitAuthDict({
+                type: "logintype",
+                foo: "bar",
+            });
+        });
+
+        ia.attemptAuth().then(function(res) {
+            expect(res).toBe(requestRes);
+            expect(doRequest.calls.length).toEqual(2);
+            expect(startAuthStage.calls.length).toEqual(1);
+        }).catch(utils.failTest).done(done);
+    });
+});
@@ -51,9 +51,10 @@ describe("MatrixClient", function() {
        // }
        // items are popped off when processed and block if no items left.
    ];
+    var accept_keepalives;
    var pendingLookup = null;
    function httpReq(cb, method, path, qp, data, prefix) {
-        if (path === KEEP_ALIVE_PATH) {
+        if (path === KEEP_ALIVE_PATH && accept_keepalives) {
            return q();
        }
        var next = httpLookups.shift();
@@ -103,6 +104,7 @@ describe("MatrixClient", function() {
            if (next.error) {
                return q.reject({
                    errcode: next.error.errcode,
+                    httpStatus: next.error.httpStatus,
                    name: next.error.errcode,
                    message: "Expected testing error",
                    data: next.error
@@ -143,8 +145,10 @@ describe("MatrixClient", function() {
        client._http.authedRequest.andCallFake(httpReq);
        client._http.authedRequestWithPrefix.andCallFake(httpReq);
        client._http.requestWithPrefix.andCallFake(httpReq);
+        client._http.request.andCallFake(httpReq);

        // set reasonable working defaults
+        accept_keepalives = true;
        pendingLookup = null;
        httpLookups = [];
        httpLookups.push(PUSH_RULES_RESPONSE);
@@ -204,6 +208,44 @@ describe("MatrixClient", function() {
        });
    });

+    describe("getOrCreateFilter", function() {
+        it("should POST createFilter if no id is present in localStorage", function() {
+        });
+        it("should use an existing filter if id is present in localStorage", function() {
+        });
+        it("should handle localStorage filterId missing from the server", function(done) {
+            function getFilterName(userId, suffix) {
+                // scope this on the user ID because people may login on many accounts
+                // and they all need to be stored!
+                return "FILTER_SYNC_" + userId + (suffix ? "_" + suffix : "");
+            }
+            var invalidFilterId = 'invalidF1lt3r';
+            httpLookups = [];
+            httpLookups.push({
+                method: "GET",
+                path: FILTER_PATH + '/' + invalidFilterId,
+                error: {
+                    errcode: "M_UNKNOWN",
+                    name: "M_UNKNOWN",
+                    message: "No row found",
+                    data: { errcode: "M_UNKNOWN", error: "No row found" },
+                    httpStatus: 404
+                }
+            });
+            httpLookups.push(FILTER_RESPONSE);
+            store.getFilterIdByName.andReturn(invalidFilterId);
+
+            var filterName = getFilterName(client.credentials.userId);
+            client.store.setFilterIdByName(filterName, invalidFilterId);
+            var filter = new sdk.Filter(client.credentials.userId);
+
+            client.getOrCreateFilter(filterName, filter).then(function(filterId) {
+                expect(filterId).toEqual(FILTER_RESPONSE.data.filter_id);
+                done();
+            });
+        });
+    });
+
    describe("retryImmediately", function() {
        it("should return false if there is no request waiting", function() {
            client.startClient();
@@ -250,6 +292,8 @@ describe("MatrixClient", function() {
                        true, "retryImmediately returned false"
                    );
                    jasmine.Clock.tick(1);
+                } else if (state === "RECONNECTING" && httpLookups.length > 0) {
+                    jasmine.Clock.tick(10000);
                } else if (state === "SYNCING" && httpLookups.length === 0) {
                    client.removeListener("sync", syncListener);
                    done();
@@ -329,17 +373,25 @@ describe("MatrixClient", function() {
        it("should transition ERROR -> PREPARED after /sync if prev failed",
        function(done) {
            var expectedStates = [];
+            accept_keepalives = false;
            httpLookups = [];
            httpLookups.push(PUSH_RULES_RESPONSE);
            httpLookups.push(FILTER_RESPONSE);
            httpLookups.push({
                method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" }
            });
+            httpLookups.push({
+                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+            });
+            httpLookups.push({
+                method: "GET", path: KEEP_ALIVE_PATH, data: {}
+            });
            httpLookups.push({
                method: "GET", path: "/sync", data: SYNC_DATA
            });

-            expectedStates.push(["ERROR", null]);
+            expectedStates.push(["RECONNECTING", null]);
+            expectedStates.push(["ERROR", "RECONNECTING"]);
            expectedStates.push(["PREPARED", "ERROR"]);
            client.on("sync", syncChecker(expectedStates, done));
            client.startClient();
@@ -354,14 +406,19 @@ describe("MatrixClient", function() {
        });

        it("should transition SYNCING -> ERROR after a failed /sync", function(done) {
+            accept_keepalives = false;
            var expectedStates = [];
            httpLookups.push({
                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
            });
+            httpLookups.push({
+                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+            });

            expectedStates.push(["PREPARED", null]);
            expectedStates.push(["SYNCING", "PREPARED"]);
-            expectedStates.push(["ERROR", "SYNCING"]);
+            expectedStates.push(["RECONNECTING", "SYNCING"]);
+            expectedStates.push(["ERROR", "RECONNECTING"]);
            client.on("sync", syncChecker(expectedStates, done));
            client.startClient();
        });
@@ -394,18 +451,23 @@ describe("MatrixClient", function() {
            client.startClient();
        });

-        it("should transition ERROR -> ERROR if multiple /sync fails", function(done) {
+        it("should transition ERROR -> ERROR if keepalive keeps failing", function(done) {
+            accept_keepalives = false;
            var expectedStates = [];
            httpLookups.push({
                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
            });
            httpLookups.push({
-                method: "GET", path: "/sync", error: { errcode: "NONONONONO" }
+                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
+            });
+            httpLookups.push({
+                method: "GET", path: KEEP_ALIVE_PATH, error: { errcode: "KEEPALIVE_FAIL" }
            });

            expectedStates.push(["PREPARED", null]);
            expectedStates.push(["SYNCING", "PREPARED"]);
-            expectedStates.push(["ERROR", "SYNCING"]);
+            expectedStates.push(["RECONNECTING", "SYNCING"]);
+            expectedStates.push(["ERROR", "RECONNECTING"]);
            expectedStates.push(["ERROR", "ERROR"]);
            client.on("sync", syncChecker(expectedStates, done));
            client.startClient();
				`@@ -1 +0,0 @@`
				`Release builds and development builds will reside here.`