v2.1.0

event.getPushRules() may return null (for better or worse...).
Use client.getPushRulesForEvent which will calculate them if they
haven't already been calculated.

Fixes https://github.com/vector-im/riot-web/issues/10269

CHANGELOG.md

@@ -1,3 +1,60 @@
+Changes in [2.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.1.0) (2019-07-08)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.1.0-rc.1...v2.1.0)
+
+ * Fix exception whilst syncing
+   [\#979](https://github.com/matrix-org/matrix-js-sdk/pull/979)
+
+Changes in [2.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.1.0-rc.1) (2019-07-03)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1...v2.1.0-rc.1)
+
+ * Handle self read receipts for fixing e2e notification counts
+   [\#974](https://github.com/matrix-org/matrix-js-sdk/pull/974)
+ * Add redacts field to event.toJSON
+   [\#973](https://github.com/matrix-org/matrix-js-sdk/pull/973)
+ * Handle associated event send failures
+   [\#972](https://github.com/matrix-org/matrix-js-sdk/pull/972)
+ * Remove irrelevant debug line from timeline handling
+   [\#971](https://github.com/matrix-org/matrix-js-sdk/pull/971)
+ * Handle relations in encrypted rooms
+   [\#969](https://github.com/matrix-org/matrix-js-sdk/pull/969)
+ * Relations endpoint support
+   [\#967](https://github.com/matrix-org/matrix-js-sdk/pull/967)
+ * Disable event encryption for reactions
+   [\#968](https://github.com/matrix-org/matrix-js-sdk/pull/968)
+ * Change the known safe room version to version 4
+   [\#966](https://github.com/matrix-org/matrix-js-sdk/pull/966)
+ * Check for lazy-loading support in the spec versions instead
+   [\#965](https://github.com/matrix-org/matrix-js-sdk/pull/965)
+ * Use camelCase instead of underscore
+   [\#963](https://github.com/matrix-org/matrix-js-sdk/pull/963)
+ * Time out verification attempts after 10 minutes of inactivity
+   [\#961](https://github.com/matrix-org/matrix-js-sdk/pull/961)
+ * Don't handle key verification requests which are immediately cancelled
+   [\#962](https://github.com/matrix-org/matrix-js-sdk/pull/962)
+
+Changes in [2.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1) (2019-06-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1-rc.2...v2.0.1)
+
+ No changes since rc.2
+
+Changes in [2.0.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1-rc.2) (2019-06-18)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1-rc.1...v2.0.1-rc.2)
+
+ * return 'sending' status for an event that is only locally redacted
+   [\#960](https://github.com/matrix-org/matrix-js-sdk/pull/960)
+ * Key verification request fixes
+   [\#954](https://github.com/matrix-org/matrix-js-sdk/pull/954)
+ * Add flag to force saving sync store
+   [\#956](https://github.com/matrix-org/matrix-js-sdk/pull/956)
+ * Expose the inhibit_login flag to register
+   [\#953](https://github.com/matrix-org/matrix-js-sdk/pull/953)
+ *  Support redactions and relations of/with unsent events.
+   [\#947](https://github.com/matrix-org/matrix-js-sdk/pull/947)
+
 Changes in [2.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1-rc.1) (2019-06-12)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.0...v2.0.1-rc.1)

README.md

@@ -296,7 +296,7 @@ Then visit ``http://localhost:8005`` to see the API docs.
 End-to-end encryption support
 =============================
 
-The SDK supports end-to-end encryption via the and Megolm protocols, using
+The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
 [libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the 
 application to make libolm available, via the ``Olm`` global.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "2.0.1-rc.1",
+  "version": "2.1.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {

spec/unit/scheduler.spec.js

@@ -48,7 +48,7 @@ describe("MatrixScheduler", function() {
         clock.uninstall();
     });
 
-    it("should process events in a queue in a FIFO manner", function(done) {
+    it("should process events in a queue in a FIFO manner", async function() {
         retryFn = function() {
             return 0;
         };
@@ -57,28 +57,30 @@ describe("MatrixScheduler", function() {
         };
         const deferA = Promise.defer();
         const deferB = Promise.defer();
-        let resolvedA = false;
+        let yieldedA = false;
         scheduler.setProcessFunction(function(event) {
-            if (resolvedA) {
+            if (yieldedA) {
                 expect(event).toEqual(eventB);
                 return deferB.promise;
             } else {
+                yieldedA = true;
                 expect(event).toEqual(eventA);
                 return deferA.promise;
             }
         });
-        scheduler.queueEvent(eventA);
-        scheduler.queueEvent(eventB).done(function() {
-            expect(resolvedA).toBe(true);
-            done();
-        });
-        deferA.resolve({});
-        resolvedA = true;
-        deferB.resolve({});
+        const abPromise = Promise.all([
+            scheduler.queueEvent(eventA),
+            scheduler.queueEvent(eventB),
+        ]);
+        deferB.resolve({b: true});
+        deferA.resolve({a: true});
+        const [a, b] = await abPromise;
+        expect(a.a).toEqual(true);
+        expect(b.b).toEqual(true);
     });
 
     it("should invoke the retryFn on failure and wait the amount of time specified",
-    function(done) {
+    async function() {
         const waitTimeMs = 1500;
         const retryDefer = Promise.defer();
         retryFn = function() {
@@ -97,24 +99,26 @@ describe("MatrixScheduler", function() {
                 return defer.promise;
             } else if (procCount === 2) {
                 // don't care about this defer
-                return Promise.defer().promise;
+                return new Promise();
             }
             expect(procCount).toBeLessThan(3);
         });
 
         scheduler.queueEvent(eventA);
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
         defer.reject({});
-        retryDefer.promise.done(function() {
-            expect(procCount).toEqual(1);
-            clock.tick(waitTimeMs);
-            expect(procCount).toEqual(2);
-            done();
-        });
+        await retryDefer.promise;
+        expect(procCount).toEqual(1);
+        clock.tick(waitTimeMs);
+        await Promise.resolve();
+        expect(procCount).toEqual(2);
     });
 
     it("should give up if the retryFn on failure returns -1 and try the next event",
-    function(done) {
+    async function() {
         // Queue A & B.
         // Reject A and return -1 on retry.
         // Expect B to be tried next and the promise for A to be rejected.
@@ -122,8 +126,8 @@ describe("MatrixScheduler", function() {
             return -1;
         };
         queueFn = function() {
- return "yep";
-};
+            return "yep";
+        };
 
         const deferA = Promise.defer();
         const deferB = Promise.defer();
@@ -142,13 +146,17 @@ describe("MatrixScheduler", function() {
 
         const globalA = scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
-
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
         deferA.reject({});
-        globalA.catch(function() {
+        try {
+            await globalA;
+        } catch(err) {
+            await Promise.resolve();
             expect(procCount).toEqual(2);
-            done();
-        });
+        }
     });
 
     it("should treat each queue separately", function(done) {
@@ -300,7 +308,11 @@ describe("MatrixScheduler", function() {
                 expect(ev).toEqual(eventA);
                 return defer.promise;
             });
-            expect(procCount).toEqual(1);
+            // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+            // wait just long enough before it does
+            Promise.resolve().then(() => {
+                expect(procCount).toEqual(1);
+            });
         });
 
         it("should not call the processFn if there are no queued events", function() {

src/ReEmitter.js

@@ -34,12 +34,18 @@ export default class Reemitter {
     }
 
     reEmit(source, eventNames) {
+        // We include the source as the last argument for event handlers which may need it,
+        // such as read receipt listeners on the client class which won't have the context
+        // of the room.
+        const forSource = (handler, ...args) => {
+            handler(...args, source);
+        };
         for (const eventName of eventNames) {
             if (this.boundHandlers[eventName] === undefined) {
                 this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
             }
-            const boundHandler = this.boundHandlers[eventName];
 
+            const boundHandler = forSource.bind(this, this.boundHandlers[eventName]);
             source.on(eventName, boundHandler);
         }
     }

src/base-apis.js

@@ -151,13 +151,14 @@ MatrixBaseApis.prototype.isUsernameAvailable = function(username) {
  *     threepid uses during registration in the ID server. Set 'msisdn' to
  *     true to bind msisdn.
  * @param {string} guestAccessToken
+ * @param {string} inhibitLogin
  * @param {module:client.callback} callback Optional.
  * @return {module:client.Promise} Resolves: TODO
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
 MatrixBaseApis.prototype.register = function(
     username, password,
-    sessionId, auth, bindThreepids, guestAccessToken,
+    sessionId, auth, bindThreepids, guestAccessToken, inhibitLogin,
     callback,
 ) {
     // backwards compat
@@ -166,6 +167,10 @@ MatrixBaseApis.prototype.register = function(
     } else if (bindThreepids === null || bindThreepids === undefined) {
         bindThreepids = {};
     }
+    if (typeof inhibitLogin === 'function') {
+        callback = inhibitLogin;
+        inhibitLogin = undefined;
+    }
 
     if (auth === undefined || auth === null) {
         auth = {};
@@ -192,6 +197,9 @@ MatrixBaseApis.prototype.register = function(
     if (guestAccessToken !== undefined && guestAccessToken !== null) {
         params.guest_access_token = guestAccessToken;
     }
+    if (inhibitLogin !== undefined && inhibitLogin !== null) {
+        params.inhibit_login = inhibitLogin;
+    }
     // Temporary parameter added to make the register endpoint advertise
     // msisdn flows. This exists because there are clients that break
     // when given stages they don't recognise. This parameter will cease
@@ -430,6 +438,35 @@ MatrixBaseApis.prototype.createRoom = function(options, callback) {
         callback, "POST", "/createRoom", undefined, options,
     );
 };
+/**
+ * Fetches relations for a given event
+ * @param {string} roomId the room of the event
+ * @param {string} eventId the id of the event
+ * @param {string} relationType the rel_type of the relations requested
+ * @param {string} eventType the event type of the relations requested
+ * @param {Object} opts options with optional values for the request.
+ * @param {Object} opts.from the pagination token returned from a previous request as `next_batch` to return following relations.
+ * @return {Object} the response, with chunk and next_batch.
+ */
+MatrixBaseApis.prototype.fetchRelations =
+    async function(roomId, eventId, relationType, eventType, opts) {
+    const queryParams = {};
+    if (opts.from) {
+        queryParams.from = opts.from;
+    }
+    const queryString = utils.encodeParams(queryParams);
+    const path = utils.encodeUri(
+        "/rooms/$roomId/relations/$eventId/$relationType/$eventType?" + queryString, {
+            $roomId: roomId,
+            $eventId: eventId,
+            $relationType: relationType,
+            $eventType: eventType,
+        });
+    const response = await this._http.authedRequestWithPrefix(
+        undefined, "GET", path, null, null, httpApi.PREFIX_UNSTABLE,
+    );
+    return response;
+};
 
 /**
  * @param {string} roomId

src/client.js

@@ -278,6 +278,48 @@ function MatrixClient(opts) {
             }
         }
     });
+
+    // Like above, we have to listen for read receipts from ourselves in order to
+    // correctly handle notification counts on encrypted rooms.
+    // This fixes https://github.com/vector-im/riot-web/issues/9421
+    this.on("Room.receipt", (event, room) => {
+        if (room && this.isRoomEncrypted(room.roomId)) {
+            // Figure out if we've read something or if it's just informational
+            const content = event.getContent();
+            const isSelf = Object.keys(content).filter(eid => {
+                return Object.keys(content[eid]['m.read']).includes(this.getUserId());
+            }).length > 0;
+
+            if (!isSelf) return;
+
+            // Work backwards to determine how many events are unread. We also set
+            // a limit for how back we'll look to avoid spinning CPU for too long.
+            // If we hit the limit, we assume the count is unchanged.
+            const maxHistory = 20;
+            const events = room.getLiveTimeline().getEvents();
+
+            let highlightCount = 0;
+
+            for (let i = events.length - 1; i >= 0; i--) {
+                if (i === events.length - maxHistory) return; // limit reached
+
+                const event = events[i];
+
+                if (room.hasUserReadEvent(this.getUserId(), event.getId())) {
+                    // If the user has read the event, then the counting is done.
+                    break;
+                }
+
+                highlightCount += this.getPushActionsForEvent(
+                    event,
+                ).tweaks.highlight ? 1 : 0;
+            }
+
+            // Note: we don't need to handle 'total' notifications because the counts
+            // will come from the server.
+            room.setUnreadNotificationCount("highlight", highlightCount);
+        }
+    });
 }
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
@@ -734,19 +776,19 @@ async function _setDeviceVerification(
  * Request a key verification from another user.
  *
  * @param {string} userId the user to request verification with
- * @param {Array} devices array of device IDs to send requests to.  Defaults to
- *    all devices owned by the user
  * @param {Array} methods array of verification methods to use.  Defaults to
  *    all known methods
+ * @param {Array} devices array of device IDs to send requests to.  Defaults to
+ *    all devices owned by the user
  *
  * @returns {Promise<module:crypto/verification/Base>} resolves to a verifier
  *    when the request is accepted by the other user
  */
-MatrixClient.prototype.requestVerification = function(userId, devices, methods) {
+MatrixClient.prototype.requestVerification = function(userId, methods, devices) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
-    return this._crypto.requestVerification(userId, devices);
+    return this._crypto.requestVerification(userId, methods, devices);
 };
 
 /**
@@ -1719,6 +1761,9 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
         txnId = this.makeTxnId();
     }
 
+    // we always construct a MatrixEvent when sending because the store and
+    // scheduler use them. We'll extract the params back out if it turns out
+    // the client has no scheduler or store.
     const localEvent = new MatrixEvent(Object.assign(eventObject, {
         event_id: "~" + roomId + ":" + txnId,
         user_id: this.credentials.userId,
@@ -1726,13 +1771,23 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
         origin_server_ts: new Date().getTime(),
     }));
 
+    const room = this.getRoom(roomId);
+
+    // if this is a relation or redaction of an event
+    // that hasn't been sent yet (e.g. with a local id starting with a ~)
+    // then listen for the remote echo of that event so that by the time
+    // this event does get sent, we have the correct event_id
+    const targetId = localEvent.getAssociatedId();
+    if (targetId && targetId.startsWith("~")) {
+        const target = room.getPendingEvents().find(e => e.getId() === targetId);
+        target.once("Event.localEventIdReplaced", () => {
+            localEvent.updateAssociatedId(target.getId());
+        });
+    }
+
     const type = localEvent.getType();
     logger.log(`sendEvent of type ${type} in ${roomId} with txnId ${txnId}`);
 
-    // we always construct a MatrixEvent when sending because the store and
-    // scheduler use them. We'll extract the params back out if it turns out
-    // the client has no scheduler or store.
-    const room = this.getRoom(roomId);
     localEvent._txnId = txnId;
     localEvent.setStatus(EventStatus.SENDING);
 
@@ -1850,6 +1905,20 @@ function _encryptEventIfNeeded(client, event, room) {
         return null;
     }
 
+    if (event.getType() === "m.reaction") {
+        // For reactions, there is a very little gained by encrypting the entire
+        // event, as relation data is already kept in the clear. Event
+        // encryption for a reaction effectively only obscures the event type,
+        // but the purpose is still obvious from the relation data, so nothing
+        // is really gained. It also causes quite a few problems, such as:
+        //   * triggers notifications via default push rules
+        //   * prevents server-side bundling for reactions
+        // The reaction key / content / emoji value does warrant encrypting, but
+        // this will be handled separately by encrypting just this value.
+        // See https://github.com/matrix-org/matrix-doc/pull/1849#pullrequestreview-248763642
+        return null;
+    }
+
     if (!client._crypto) {
         throw new Error(
             "This room is configured to use encryption, but your client does " +
@@ -1859,6 +1928,21 @@ function _encryptEventIfNeeded(client, event, room) {
 
     return client._crypto.encryptEvent(event, room);
 }
+/**
+ * Returns the eventType that should be used taking encryption into account
+ * for a given eventType.
+ * @param {MatrixClient} client the client
+ * @param {string} roomId the room for the events `eventType` relates to
+ * @param {string} eventType the event type
+ * @return {string} the event type taking encryption into account
+ */
+function _getEncryptedIfNeededEventType(client, roomId, eventType) {
+    if (eventType === "m.reaction") {
+        return eventType;
+    }
+    const isEncrypted = client.isRoomEncrypted(roomId);
+    return isEncrypted ? "m.room.encrypted" : eventType;
+}
 
 function _updatePendingEventStatus(room, event, newStatus) {
     if (room) {
@@ -1886,7 +1970,7 @@ function _sendEventHttpRequest(client, event) {
             pathTemplate = "/rooms/$roomId/state/$eventType/$stateKey";
         }
         path = utils.encodeUri(pathTemplate, pathParams);
-    } else if (event.getType() === "m.room.redaction") {
+    } else if (event.isRedaction()) {
         const pathTemplate = `/rooms/$roomId/redact/$redactsEventId/$txnId`;
         path = utils.encodeUri(pathTemplate, Object.assign({
             $redactsEventId: event.event.redacts,
@@ -3912,9 +3996,13 @@ MatrixClient.prototype.doesServerSupportLazyLoading = async function() {
                 prefix: '',
             },
         );
+
+        const versions = response["versions"];
         const unstableFeatures = response["unstable_features"];
+
         this._serverSupportsLazyLoading =
-            unstableFeatures && unstableFeatures["m.lazy_load_members"];
+            (versions && versions.includes("r0.5.0"))
+            || (unstableFeatures && unstableFeatures["m.lazy_load_members"]);
     }
     return this._serverSupportsLazyLoading;
 };
@@ -3948,6 +4036,41 @@ MatrixClient.prototype.getCanResetTimelineCallback = function() {
     return this._canResetTimelineCallback;
 };
 
+/**
+ * Returns relations for a given event. Handles encryption transparently,
+ * with the caveat that the amount of events returned might be 0, even though you get a nextBatch.
+ * When the returned promise resolves, all messages should have finished trying to decrypt.
+ * @param {string} roomId the room of the event
+ * @param {string} eventId the id of the event
+ * @param {string} relationType the rel_type of the relations requested
+ * @param {string} eventType the event type of the relations requested
+ * @param {Object} opts options with optional values for the request.
+ * @param {Object} opts.from the pagination token returned from a previous request as `nextBatch` to return following relations.
+ * @return {Object} an object with `events` as `MatrixEvent[]` and optionally `nextBatch` if more relations are available.
+ */
+MatrixClient.prototype.relations =
+async function(roomId, eventId, relationType, eventType, opts = {}) {
+    const fetchedEventType = _getEncryptedIfNeededEventType(this, roomId, eventType);
+    const result = await this.fetchRelations(
+        roomId,
+        eventId,
+        relationType,
+        fetchedEventType,
+        opts);
+
+    let events = result.chunk.map(this.getEventMapper());
+    if (fetchedEventType === "m.room.encrypted") {
+        await Promise.all(events.map(e => {
+            return new Promise(resolve => e.once("Event.decrypted", resolve));
+        }));
+        events = events.filter(e => e.getType() === eventType);
+    }
+    return {
+        events,
+        nextBatch: result.next_batch,
+    };
+};
+
 function setupCallEventHandler(client) {
     const candidatesByCall = {
         // callId: [Candidate]

src/crypto/index.js

@@ -1648,6 +1648,11 @@ Crypto.prototype._onRoomKeyEvent = function(event) {
  * @param {module:models/event.MatrixEvent} event verification request event
  */
 Crypto.prototype._onKeyVerificationRequest = function(event) {
+    if (event.isCancelled()) {
+        logger.warn("Ignoring flagged verification request from " + event.getSender());
+        return;
+    }
+
     const content = event.getContent();
     if (!("from_device" in content) || typeof content.from_device !== "string"
         || !("transaction_id" in content) || typeof content.from_device !== "string"
@@ -1667,6 +1672,11 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
     }
 
     const sender = event.getSender();
+    if (sender === this._userId && content.from_device === this._deviceId) {
+        // ignore requests from ourselves, because it doesn't make sense for a
+        // device to verify itself
+        return;
+    }
     if (this._verificationTransactions.has(sender)) {
         if (this._verificationTransactions.get(sender).has(content.transaction_id)) {
             // transaction already exists: cancel it and drop the existing
@@ -1719,7 +1729,7 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
             },
         );
     } else {
-        // notify the application that of the verification request, so it can
+        // notify the application of the verification request, so it can
         // decide what to do with it
         const request = {
             event: event,
@@ -1759,6 +1769,11 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
  * @param {module:models/event.MatrixEvent} event verification start event
  */
 Crypto.prototype._onKeyVerificationStart = function(event) {
+    if (event.isCancelled()) {
+        logger.warn("Ignoring flagged verification start from " + event.getSender());
+        return;
+    }
+
     const sender = event.getSender();
     const content = event.getContent();
     const transactionId = content.transaction_id;

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -702,7 +702,7 @@ function promiseifyTxn(txn) {
             if (txn._mx_abortexception !== undefined) {
                 reject(txn._mx_abortexception);
             } else {
-                localStorage.log("Error performing indexeddb txn", event);
+                logger.log("Error performing indexeddb txn", event);
                 reject(event.target.error);
             }
         };

src/crypto/verification/Base.js

@@ -22,6 +22,9 @@ limitations under the License.
 import {MatrixEvent} from '../../models/event';
 import {EventEmitter} from 'events';
 import logger from '../../logger';
+import {newTimeoutError} from "./Error";
+
+const timeoutException = new Error("Verification timed out");
 
 export default class VerificationBase extends EventEmitter {
     /**
@@ -60,9 +63,27 @@ export default class VerificationBase extends EventEmitter {
         this.transactionId = transactionId;
         this.startEvent = startEvent;
         this.request = request;
+        this.cancelled = false;
         this._parent = parent;
         this._done = false;
         this._promise = null;
+        this._transactionTimeoutTimer = null;
+
+        // At this point, the verification request was received so start the timeout timer.
+        this._resetTimer();
+    }
+
+    _resetTimer() {
+        console.log("Refreshing/starting the verification transaction timeout timer");
+        if (this._transactionTimeoutTimer !== null) {
+            clearTimeout(this._transactionTimeoutTimer);
+        }
+        setTimeout(() => {
+           if (!this._done && !this.cancelled) {
+               console.log("Triggering verification timeout");
+               this.cancel(timeoutException);
+           }
+        }, 10 * 60 * 1000); // 10 minutes
     }
 
     _sendToDevice(type, content) {
@@ -92,6 +113,7 @@ export default class VerificationBase extends EventEmitter {
         } else if (e.getType() === this._expectedEvent) {
             this._expectedEvent = undefined;
             this._rejectEvent = undefined;
+            this._resetTimer();
             this._resolveEvent(e);
         } else {
             this._expectedEvent = undefined;
@@ -116,10 +138,14 @@ export default class VerificationBase extends EventEmitter {
 
     cancel(e) {
         if (!this._done) {
+            this.cancelled = true;
             if (this.userId && this.deviceId && this.transactionId) {
                 // send a cancellation to the other user (if it wasn't
                 // cancelled by the other user)
-                if (e instanceof MatrixEvent) {
+                if (e === timeoutException) {
+                    const timeoutEvent = newTimeoutError();
+                    this._sendToDevice(timeoutEvent.getType(), timeoutEvent.getContent());
+                } else if (e instanceof MatrixEvent) {
                     const sender = e.getSender();
                     if (sender !== this.userId) {
                         const content = e.getContent();
@@ -146,7 +172,9 @@ export default class VerificationBase extends EventEmitter {
                 }
             }
             if (this._promise !== null) {
-                this._reject(e);
+                // when we cancel without a promise, we end up with a promise
+                // but no reject function. If cancel is called again, we'd error.
+                if (this._reject) this._reject(e);
             } else {
                 this._promise = Promise.reject(e);
             }
@@ -177,6 +205,7 @@ export default class VerificationBase extends EventEmitter {
         });
         if (this._doVerification && !this._started) {
             this._started = true;
+            this._resetTimer(); // restart the timeout
             Promise.resolve(this._doVerification())
                 .then(this.done.bind(this), this.cancel.bind(this));
         }

src/models/event-timeline-set.js

@@ -20,6 +20,7 @@ limitations under the License.
 const EventEmitter = require("events").EventEmitter;
 const utils = require("../utils");
 const EventTimeline = require("./event-timeline");
+import {EventStatus} from "./event";
 import logger from '../../src/logger';
 import Relations from './relations';
 
@@ -437,7 +438,6 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
         if (backwardsIsLive || forwardsIsLive) {
             // The live timeline should never be spliced into a non-live position.
             // We use independent logging to better discover the problem at a glance.
-            logger.warn({backwardsIsLive, forwardsIsLive}); // debugging
             if (backwardsIsLive) {
                 logger.warn(
                     "Refusing to set a preceding existingTimeLine on our " +
@@ -749,6 +749,10 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
         return;
     }
 
+    if (event.isRedacted() || event.status === EventStatus.CANCELLED) {
+        return;
+    }
+
     // If the event is currently encrypted, wait until it has been decrypted.
     if (event.isBeingDecrypted()) {
         event.once("Event.decrypted", () => {

src/models/event.js

@@ -31,7 +31,7 @@ import logger from '../../src/logger';
  * @readonly
  * @enum {string}
  */
-module.exports.EventStatus = {
+const EventStatus = {
     /** The event was not sent and will no longer be retried. */
     NOT_SENT: "not_sent",
 
@@ -49,6 +49,7 @@ module.exports.EventStatus = {
     /** The event was cancelled before it was successfully sent. */
     CANCELLED: "cancelled",
 };
+module.exports.EventStatus = EventStatus;
 
 const interns = {};
 function intern(str) {
@@ -124,7 +125,8 @@ module.exports.MatrixEvent = function MatrixEvent(
     this.forwardLooking = true;
     this._pushActions = null;
     this._replacingEvent = null;
-    this._locallyRedacted = false;
+    this._localRedactionEvent = null;
+    this._isCancelled = false;
 
     this._clearEvent = {};
 
@@ -229,7 +231,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Object} The event content JSON, or an empty object.
      */
     getOriginalContent: function() {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return {};
         }
         return this._clearEvent.content || this.event.content || {};
@@ -243,7 +245,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Object} The event content JSON, or an empty object.
      */
     getContent: function() {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return {};
         } else if (this._replacingEvent) {
             return this._replacingEvent.getContent()["m.new_content"] || {};
@@ -673,20 +675,20 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     unmarkLocallyRedacted: function() {
-        const value = this._locallyRedacted;
-        this._locallyRedacted = false;
+        const value = this._localRedactionEvent;
+        this._localRedactionEvent = null;
         if (this.event.unsigned) {
             this.event.unsigned.redacted_because = null;
         }
-        return value;
+        return !!value;
     },
 
     markLocallyRedacted: function(redactionEvent) {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return;
         }
         this.emit("Event.beforeRedaction", this, redactionEvent);
-        this._locallyRedacted = true;
+        this._localRedactionEvent = redactionEvent;
         if (!this.event.unsigned) {
             this.event.unsigned = {};
         }
@@ -706,7 +708,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             throw new Error("invalid redaction_event in makeRedacted");
         }
 
-        this._locallyRedacted = false;
+        this._localRedactionEvent = null;
 
         this.emit("Event.beforeRedaction", this, redaction_event);
 
@@ -753,6 +755,15 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return Boolean(this.getUnsigned().redacted_because);
     },
 
+    /**
+     * Check if this event is a redaction of another event
+     *
+     * @return {boolean} True if this event is a redaction
+     */
+    isRedaction: function() {
+        return this.getType() === "m.room.redaction";
+    },
+
     /**
      * Get the push actions, if known, for this event
      *
@@ -776,9 +787,26 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @param {Object} event the object to assign to the `event` property
      */
     handleRemoteEcho: function(event) {
+        const oldUnsigned = this.getUnsigned();
+        const oldId = this.getId();
         this.event = event;
+        // if this event was redacted before it was sent, it's locally marked as redacted.
+        // At this point, we've received the remote echo for the event, but not yet for
+        // the redaction that we are sending ourselves. Preserve the locally redacted
+        // state by copying over redacted_because so we don't get a flash of
+        // redacted, not-redacted, redacted as remote echos come in
+        if (oldUnsigned.redacted_because) {
+            if (!this.event.unsigned) {
+                this.event.unsigned = {};
+            }
+            this.event.unsigned.redacted_because = oldUnsigned.redacted_because;
+        }
         // successfully sent.
         this.setStatus(null);
+        if (this.getId() !== oldId) {
+            // emit the event if it changed
+            this.emit("Event.localEventIdReplaced", this);
+        }
     },
 
     /**
@@ -801,6 +829,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         this.emit("Event.status", this, status);
     },
 
+    replaceLocalEventId(eventId) {
+        this.event.event_id = eventId;
+        this.emit("Event.localEventIdReplaced", this);
+    },
+
     /**
      * Get whether the event is a relation event, and of a given type if
      * `relType` is passed in.
@@ -836,7 +869,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {
-        if (this.isRedacted()) {
+        // don't allow redacted events to be replaced.
+        // if newEvent is null we allow to go through though,
+        // as with local redaction, the replacing event might get
+        // cancelled, which should be reflected on the target event.
+        if (this.isRedacted() && newEvent) {
             return;
         }
         if (this._replacingEvent !== newEvent) {
@@ -846,16 +883,19 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     /**
-     * Returns the status of the event, or the replacing event in case `makeReplace` has been called.
+     * Returns the status of any associated edit or redaction
+     * (not for reactions/annotations as their local echo doesn't affect the orignal event),
+     * or else the status of the event.
      *
      * @return {EventStatus}
      */
-    replacementOrOwnStatus() {
+    getAssociatedStatus() {
         if (this._replacingEvent) {
             return this._replacingEvent.status;
-        } else {
-            return this.status;
+        } else if (this._localRedactionEvent) {
+            return this._localRedactionEvent.status;
         }
+        return this.status;
     },
 
     /**
@@ -869,6 +909,8 @@ utils.extend(module.exports.MatrixEvent.prototype, {
 
     /**
      * Returns the event replacing the content of this event, if any.
+     * Replacements are aggregated on the server, so this would only
+     * return an event in case it came down the sync, or for local echo of edits.
      *
      * @return {MatrixEvent?}
      */
@@ -876,6 +918,73 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return this._replacingEvent;
     },
 
+    /**
+     * Returns the event that wants to redact this event, but hasn't been sent yet.
+     * @return {MatrixEvent} the event
+     */
+    localRedactionEvent() {
+        return this._localRedactionEvent;
+    },
+
+    /**
+     * For relations and redactions, returns the event_id this event is referring to.
+     *
+     * @return {string?}
+     */
+    getAssociatedId() {
+        const relation = this.getRelation();
+        if (relation) {
+            return relation.event_id;
+        } else if (this.isRedaction()) {
+            return this.event.redacts;
+        }
+    },
+
+    /**
+     * Checks if this event is associated with another event. See `getAssociatedId`.
+     *
+     * @return {bool}
+     */
+    hasAssocation() {
+        return !!this.getAssociatedId();
+    },
+
+    /**
+     * Update the related id with a new one.
+     *
+     * Used to replace a local id with remote one before sending
+     * an event with a related id.
+     *
+     * @param {string} eventId the new event id
+     */
+    updateAssociatedId(eventId) {
+        const relation = this.getRelation();
+        if (relation) {
+            relation.event_id = eventId;
+        } else if (this.isRedaction()) {
+            this.event.redacts = eventId;
+        }
+    },
+
+    /**
+     * Flags an event as cancelled due to future conditions. For example, a verification
+     * request event in the same sync transaction may be flagged as cancelled to warn
+     * listeners that a cancellation event is coming down the same pipe shortly.
+     * @param {boolean} cancelled Whether the event is to be cancelled or not.
+     */
+    flagCancelled(cancelled = true) {
+        this._isCancelled = cancelled;
+    },
+
+    /**
+     * Gets whether or not the event is flagged as cancelled. See flagCancelled() for
+     * more information.
+     * @returns {boolean} True if the event is cancelled, false otherwise.
+     */
+    isCancelled() {
+        return this._isCancelled;
+    },
+
     /**
      * Summarise the event as JSON for debugging. If encrypted, include both the
      * decrypted and encrypted view of the event. This is named `toJSON` for use
@@ -895,6 +1004,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             room_id: this.getRoomId(),
         };
 
+        // if this is a redaction then attach the redacts key
+        if (this.isRedaction()) {
+            event.redacts = this.event.redacts;
+        }
+
         if (!this.isEncrypted()) {
             return event;
         }

src/models/relations.js

@@ -242,12 +242,7 @@ export default class Relations extends EventEmitter {
 
         redactedEvent.removeListener("Event.beforeRedaction", this._onBeforeRedaction);
 
-        // Dispatch a redaction event on this collection. `setTimeout` is used
-        // to wait until the next event loop iteration by which time the event
-        // has actually been marked as redacted.
-        setTimeout(() => {
-            this.emit("Relations.redaction");
-        }, 0);
+        this.emit("Relations.redaction");
     }
 
     /**

src/models/room.js

@@ -38,7 +38,7 @@ import ReEmitter from '../ReEmitter';
 // room versions which are considered okay for people to run without being asked
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
-const KNOWN_SAFE_ROOM_VERSION = '1';
+const KNOWN_SAFE_ROOM_VERSION = '4';
 const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4'];
 
 function synthesizeReceipt(userId, event, receiptType) {
@@ -797,20 +797,20 @@ Room.prototype.getAvatarUrl = function(baseUrl, width, height, resizeMethod,
  * @return {array} The room's alias as an array of strings
  */
 Room.prototype.getAliases = function() {
-    const alias_strings = [];
+    const aliasStrings = [];
 
-    const alias_events = this.currentState.getStateEvents("m.room.aliases");
-    if (alias_events) {
-        for (let i = 0; i < alias_events.length; ++i) {
-            const alias_event = alias_events[i];
-            if (utils.isArray(alias_event.getContent().aliases)) {
+    const aliasEvents = this.currentState.getStateEvents("m.room.aliases");
+    if (aliasEvents) {
+        for (let i = 0; i < aliasEvents.length; ++i) {
+            const aliasEvent = aliasEvents[i];
+            if (utils.isArray(aliasEvent.getContent().aliases)) {
                 Array.prototype.push.apply(
-                    alias_strings, alias_event.getContent().aliases,
+                    aliasStrings, aliasEvent.getContent().aliases,
                 );
             }
         }
     }
-    return alias_strings;
+    return aliasStrings;
 };
 
 /**
@@ -1032,7 +1032,7 @@ Room.prototype.removeFilteredTimelineSet = function(filter) {
  * @private
  */
 Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
-    if (event.getType() === "m.room.redaction") {
+    if (event.isRedaction()) {
         const redactId = event.event.redacts;
 
         // if we know about this event, redact its contents now.
@@ -1141,9 +1141,13 @@ Room.prototype.addPendingEvent = function(event, txnId) {
             this._aggregateNonLiveRelation(event);
         }
 
-        if (event.getType() === "m.room.redaction") {
+        if (event.isRedaction()) {
             const redactId = event.event.redacts;
-            const redactedEvent = this.getUnfilteredTimelineSet().findEventById(redactId);
+            let redactedEvent = this._pendingEventList &&
+                this._pendingEventList.find(e => e.getId() === redactId);
+            if (!redactedEvent) {
+                redactedEvent = this.getUnfilteredTimelineSet().findEventById(redactId);
+            }
             if (redactedEvent) {
                 redactedEvent.markLocallyRedacted(event);
                 this.emit("Room.redaction", event, this);
@@ -1211,7 +1215,7 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
     const oldStatus = localEvent.status;
 
     // no longer pending
-    delete this._txnToEvent[remoteEvent.transaction_id];
+    delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];
 
     // if it's in the pending list, remove it
     if (this._pendingEventList) {
@@ -1315,7 +1319,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
 
     if (newStatus == EventStatus.SENT) {
         // update the event id
-        event.event.event_id = newEventId;
+        event.replaceLocalEventId(newEventId);
 
         // if the event was already in the timeline (which will be the case if
         // opts.pendingEventOrdering==chronological), we need to update the
@@ -1329,7 +1333,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
             const idx = this._pendingEventList.findIndex(ev => ev.getId() === oldEventId);
             if (idx !== -1) {
                 const [removedEvent] = this._pendingEventList.splice(idx, 1);
-                if (removedEvent.getType() === "m.room.redaction") {
+                if (removedEvent.isRedaction()) {
                     this._revertRedactionLocalEcho(removedEvent);
                 }
             }
@@ -1415,11 +1419,11 @@ Room.prototype.addLiveEvents = function(events, duplicateStrategy) {
 
 /**
  * Removes events from this room.
- * @param {String[]} event_ids A list of event_ids to remove.
+ * @param {String[]} eventIds A list of eventIds to remove.
  */
-Room.prototype.removeEvents = function(event_ids) {
-    for (let i = 0; i < event_ids.length; ++i) {
-        this.removeEvent(event_ids[i]);
+Room.prototype.removeEvents = function(eventIds) {
+    for (let i = 0; i < eventIds.length; ++i) {
+        this.removeEvent(eventIds[i]);
     }
 };
 
@@ -1435,7 +1439,7 @@ Room.prototype.removeEvent = function(eventId) {
     for (let i = 0; i < this._timelineSets.length; i++) {
         const removed = this._timelineSets[i].removeEvent(eventId);
         if (removed) {
-            if (removed.getType() === "m.room.redaction") {
+            if (removed.isRedaction()) {
                 this._revertRedactionLocalEcho(removed);
             }
             removedAny = true;

src/scheduler.js

@@ -177,7 +177,8 @@ MatrixScheduler.RETRY_BACKOFF_RATELIMIT = function(event, attempts, err) {
  * @see module:scheduler~queueAlgorithm
  */
 MatrixScheduler.QUEUE_MESSAGES = function(event) {
-    if (event.getType() === "m.room.message") {
+    // enqueue messages or events that associate with another event (redactions and relations)
+    if (event.getType() === "m.room.message" || event.hasAssocation()) {
         // put these events in the 'message' queue.
         return "message";
     }
@@ -220,7 +221,14 @@ function _processQueue(scheduler, queueName) {
     );
     // fire the process function and if it resolves, resolve the deferred. Else
     // invoke the retry algorithm.
-    scheduler._procFn(obj.event).done(function(res) {
+
+    // First wait for a resolved promise, so the resolve handlers for
+    // the deferred of the previously sent event can run.
+    // This way enqueued relations/redactions to enqueued events can receive
+    // the remove id of their target before being sent.
+    Promise.resolve().then(() => {
+        return scheduler._procFn(obj.event);
+    }).then(function(res) {
         // remove this from the queue
         _removeNextEvent(scheduler, queueName);
         debuglog("Queue '%s' sent event %s", queueName, obj.event.getId());

src/store/indexeddb.js

@@ -198,11 +198,13 @@ IndexedDBStore.prototype.wantsSave = function() {
 
 /**
  * Possibly write data to the database.
+ *
+ * @param {bool} force True to force a save to happen
  * @return {Promise} Promise resolves after the write completes
  *     (or immediately if no write is performed)
  */
-IndexedDBStore.prototype.save = function() {
-    if (this.wantsSave()) {
+IndexedDBStore.prototype.save = function(force) {
+    if (force || this.wantsSave()) {
         return this._reallySave();
     }
     return Promise.resolve();

src/store/memory.js

@@ -335,8 +335,10 @@ module.exports.MemoryStore.prototype = {
 
     /**
      * Save does nothing as there is no backing data store.
+     * @param {bool} force True to force a save (but the memory
+     *     store still can't save anything)
      */
-    save: function() {},
+    save: function(force) {},
 
     /**
      * Startup does nothing as this store doesn't require starting up.

src/sync.js

@@ -1043,8 +1043,26 @@ SyncApi.prototype._processSyncResponse = async function(
     if (data.to_device && utils.isArray(data.to_device.events) &&
         data.to_device.events.length > 0
        ) {
+        const cancelledKeyVerificationTxns = [];
         data.to_device.events
             .map(client.getEventMapper())
+            .map((toDeviceEvent) => { // map is a cheap inline forEach
+                // We want to flag m.key.verification.start events as cancelled
+                // if there's an accompanying m.key.verification.cancel event, so
+                // we pull out the transaction IDs from the cancellation events
+                // so we can flag the verification events as cancelled in the loop
+                // below.
+                if (toDeviceEvent.getType() === "m.key.verification.cancel") {
+                    const txnId = toDeviceEvent.getContent()['transaction_id'];
+                    if (txnId) {
+                        cancelledKeyVerificationTxns.push(txnId);
+                    }
+                }
+
+                // as mentioned above, .map is a cheap inline forEach, so return
+                // the unmodified event.
+                return toDeviceEvent;
+            })
             .forEach(
                 function(toDeviceEvent) {
                     const content = toDeviceEvent.getContent();
@@ -1060,6 +1078,14 @@ SyncApi.prototype._processSyncResponse = async function(
                         return;
                     }
 
+                    if (toDeviceEvent.getType() === "m.key.verification.start"
+                        || toDeviceEvent.getType() === "m.key.verification.request") {
+                        const txnId = content['transaction_id'];
+                        if (cancelledKeyVerificationTxns.includes(txnId)) {
+                            toDeviceEvent.flagCancelled();
+                        }
+                    }
+
                     client.emit("toDeviceEvent", toDeviceEvent);
                 },
             );