v2.0.1

return 'sending' status for an event that is only locally redacted

CHANGELOG.md

@@ -1,3 +1,24 @@
+Changes in [2.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1) (2019-06-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1-rc.2...v2.0.1)
+
+ No changes since rc.2
+
+Changes in [2.0.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1-rc.2) (2019-06-18)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1-rc.1...v2.0.1-rc.2)
+
+ * return 'sending' status for an event that is only locally redacted
+   [\#960](https://github.com/matrix-org/matrix-js-sdk/pull/960)
+ * Key verification request fixes
+   [\#954](https://github.com/matrix-org/matrix-js-sdk/pull/954)
+ * Add flag to force saving sync store
+   [\#956](https://github.com/matrix-org/matrix-js-sdk/pull/956)
+ * Expose the inhibit_login flag to register
+   [\#953](https://github.com/matrix-org/matrix-js-sdk/pull/953)
+ *  Support redactions and relations of/with unsent events.
+   [\#947](https://github.com/matrix-org/matrix-js-sdk/pull/947)
+
 Changes in [2.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1-rc.1) (2019-06-12)
 ==========================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.0...v2.0.1-rc.1)

README.md

@@ -296,7 +296,7 @@ Then visit ``http://localhost:8005`` to see the API docs.
 End-to-end encryption support
 =============================
 
-The SDK supports end-to-end encryption via the and Megolm protocols, using
+The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
 [libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the 
 application to make libolm available, via the ``Olm`` global.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "2.0.1-rc.1",
+  "version": "2.0.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {

spec/unit/scheduler.spec.js

@@ -48,7 +48,7 @@ describe("MatrixScheduler", function() {
         clock.uninstall();
     });
 
-    it("should process events in a queue in a FIFO manner", function(done) {
+    it("should process events in a queue in a FIFO manner", async function() {
         retryFn = function() {
             return 0;
         };
@@ -57,28 +57,30 @@ describe("MatrixScheduler", function() {
         };
         const deferA = Promise.defer();
         const deferB = Promise.defer();
-        let resolvedA = false;
+        let yieldedA = false;
         scheduler.setProcessFunction(function(event) {
-            if (resolvedA) {
+            if (yieldedA) {
                 expect(event).toEqual(eventB);
                 return deferB.promise;
             } else {
+                yieldedA = true;
                 expect(event).toEqual(eventA);
                 return deferA.promise;
             }
         });
-        scheduler.queueEvent(eventA);
-        scheduler.queueEvent(eventB).done(function() {
-            expect(resolvedA).toBe(true);
-            done();
-        });
-        deferA.resolve({});
-        resolvedA = true;
-        deferB.resolve({});
+        const abPromise = Promise.all([
+            scheduler.queueEvent(eventA),
+            scheduler.queueEvent(eventB),
+        ]);
+        deferB.resolve({b: true});
+        deferA.resolve({a: true});
+        const [a, b] = await abPromise;
+        expect(a.a).toEqual(true);
+        expect(b.b).toEqual(true);
     });
 
     it("should invoke the retryFn on failure and wait the amount of time specified",
-    function(done) {
+    async function() {
         const waitTimeMs = 1500;
         const retryDefer = Promise.defer();
         retryFn = function() {
@@ -97,24 +99,26 @@ describe("MatrixScheduler", function() {
                 return defer.promise;
             } else if (procCount === 2) {
                 // don't care about this defer
-                return Promise.defer().promise;
+                return new Promise();
             }
             expect(procCount).toBeLessThan(3);
         });
 
         scheduler.queueEvent(eventA);
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
         defer.reject({});
-        retryDefer.promise.done(function() {
-            expect(procCount).toEqual(1);
-            clock.tick(waitTimeMs);
-            expect(procCount).toEqual(2);
-            done();
-        });
+        await retryDefer.promise;
+        expect(procCount).toEqual(1);
+        clock.tick(waitTimeMs);
+        await Promise.resolve();
+        expect(procCount).toEqual(2);
     });
 
     it("should give up if the retryFn on failure returns -1 and try the next event",
-    function(done) {
+    async function() {
         // Queue A & B.
         // Reject A and return -1 on retry.
         // Expect B to be tried next and the promise for A to be rejected.
@@ -122,8 +126,8 @@ describe("MatrixScheduler", function() {
             return -1;
         };
         queueFn = function() {
- return "yep";
-};
+            return "yep";
+        };
 
         const deferA = Promise.defer();
         const deferB = Promise.defer();
@@ -142,13 +146,17 @@ describe("MatrixScheduler", function() {
 
         const globalA = scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
-
+        // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+        // wait just long enough before it does
+        await Promise.resolve();
         expect(procCount).toEqual(1);
         deferA.reject({});
-        globalA.catch(function() {
+        try {
+            await globalA;
+        } catch(err) {
+            await Promise.resolve();
             expect(procCount).toEqual(2);
-            done();
-        });
+        }
     });
 
     it("should treat each queue separately", function(done) {
@@ -300,7 +308,11 @@ describe("MatrixScheduler", function() {
                 expect(ev).toEqual(eventA);
                 return defer.promise;
             });
-            expect(procCount).toEqual(1);
+            // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
+            // wait just long enough before it does
+            Promise.resolve().then(() => {
+                expect(procCount).toEqual(1);
+            });
         });
 
         it("should not call the processFn if there are no queued events", function() {

src/base-apis.js

@@ -151,13 +151,14 @@ MatrixBaseApis.prototype.isUsernameAvailable = function(username) {
  *     threepid uses during registration in the ID server. Set 'msisdn' to
  *     true to bind msisdn.
  * @param {string} guestAccessToken
+ * @param {string} inhibitLogin
  * @param {module:client.callback} callback Optional.
  * @return {module:client.Promise} Resolves: TODO
  * @return {module:http-api.MatrixError} Rejects: with an error response.
  */
 MatrixBaseApis.prototype.register = function(
     username, password,
-    sessionId, auth, bindThreepids, guestAccessToken,
+    sessionId, auth, bindThreepids, guestAccessToken, inhibitLogin,
     callback,
 ) {
     // backwards compat
@@ -166,6 +167,10 @@ MatrixBaseApis.prototype.register = function(
     } else if (bindThreepids === null || bindThreepids === undefined) {
         bindThreepids = {};
     }
+    if (typeof inhibitLogin === 'function') {
+        callback = inhibitLogin;
+        inhibitLogin = undefined;
+    }
 
     if (auth === undefined || auth === null) {
         auth = {};
@@ -192,6 +197,9 @@ MatrixBaseApis.prototype.register = function(
     if (guestAccessToken !== undefined && guestAccessToken !== null) {
         params.guest_access_token = guestAccessToken;
     }
+    if (inhibitLogin !== undefined && inhibitLogin !== null) {
+        params.inhibit_login = inhibitLogin;
+    }
     // Temporary parameter added to make the register endpoint advertise
     // msisdn flows. This exists because there are clients that break
     // when given stages they don't recognise. This parameter will cease

src/client.js

@@ -734,19 +734,19 @@ async function _setDeviceVerification(
  * Request a key verification from another user.
  *
  * @param {string} userId the user to request verification with
- * @param {Array} devices array of device IDs to send requests to.  Defaults to
- *    all devices owned by the user
  * @param {Array} methods array of verification methods to use.  Defaults to
  *    all known methods
+ * @param {Array} devices array of device IDs to send requests to.  Defaults to
+ *    all devices owned by the user
  *
  * @returns {Promise<module:crypto/verification/Base>} resolves to a verifier
  *    when the request is accepted by the other user
  */
-MatrixClient.prototype.requestVerification = function(userId, devices, methods) {
+MatrixClient.prototype.requestVerification = function(userId, methods, devices) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
     }
-    return this._crypto.requestVerification(userId, devices);
+    return this._crypto.requestVerification(userId, methods, devices);
 };
 
 /**
@@ -1719,6 +1719,9 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
         txnId = this.makeTxnId();
     }
 
+    // we always construct a MatrixEvent when sending because the store and
+    // scheduler use them. We'll extract the params back out if it turns out
+    // the client has no scheduler or store.
     const localEvent = new MatrixEvent(Object.assign(eventObject, {
         event_id: "~" + roomId + ":" + txnId,
         user_id: this.credentials.userId,
@@ -1726,13 +1729,23 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
         origin_server_ts: new Date().getTime(),
     }));
 
+    const room = this.getRoom(roomId);
+
+    // if this is a relation or redaction of an event
+    // that hasn't been sent yet (e.g. with a local id starting with a ~)
+    // then listen for the remote echo of that event so that by the time
+    // this event does get sent, we have the correct event_id
+    const targetId = localEvent.getAssociatedId();
+    if (targetId && targetId.startsWith("~")) {
+        const target = room.getPendingEvents().find(e => e.getId() === targetId);
+        target.once("Event.localEventIdReplaced", () => {
+            localEvent.updateAssociatedId(target.getId());
+        });
+    }
+
     const type = localEvent.getType();
     logger.log(`sendEvent of type ${type} in ${roomId} with txnId ${txnId}`);
 
-    // we always construct a MatrixEvent when sending because the store and
-    // scheduler use them. We'll extract the params back out if it turns out
-    // the client has no scheduler or store.
-    const room = this.getRoom(roomId);
     localEvent._txnId = txnId;
     localEvent.setStatus(EventStatus.SENDING);
 
@@ -1886,7 +1899,7 @@ function _sendEventHttpRequest(client, event) {
             pathTemplate = "/rooms/$roomId/state/$eventType/$stateKey";
         }
         path = utils.encodeUri(pathTemplate, pathParams);
-    } else if (event.getType() === "m.room.redaction") {
+    } else if (event.isRedaction()) {
         const pathTemplate = `/rooms/$roomId/redact/$redactsEventId/$txnId`;
         path = utils.encodeUri(pathTemplate, Object.assign({
             $redactsEventId: event.event.redacts,

src/crypto/index.js

@@ -1667,6 +1667,11 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
     }
 
     const sender = event.getSender();
+    if (sender === this._userId && content.from_device === this._deviceId) {
+        // ignore requests from ourselves, because it doesn't make sense for a
+        // device to verify itself
+        return;
+    }
     if (this._verificationTransactions.has(sender)) {
         if (this._verificationTransactions.get(sender).has(content.transaction_id)) {
             // transaction already exists: cancel it and drop the existing
@@ -1719,7 +1724,7 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
             },
         );
     } else {
-        // notify the application that of the verification request, so it can
+        // notify the application of the verification request, so it can
         // decide what to do with it
         const request = {
             event: event,

src/models/event-timeline-set.js

@@ -20,6 +20,7 @@ limitations under the License.
 const EventEmitter = require("events").EventEmitter;
 const utils = require("../utils");
 const EventTimeline = require("./event-timeline");
+import {EventStatus} from "./event";
 import logger from '../../src/logger';
 import Relations from './relations';
 
@@ -749,6 +750,10 @@ EventTimelineSet.prototype.aggregateRelations = function(event) {
         return;
     }
 
+    if (event.isRedacted() || event.status === EventStatus.CANCELLED) {
+        return;
+    }
+
     // If the event is currently encrypted, wait until it has been decrypted.
     if (event.isBeingDecrypted()) {
         event.once("Event.decrypted", () => {

src/models/event.js

@@ -31,7 +31,7 @@ import logger from '../../src/logger';
  * @readonly
  * @enum {string}
  */
-module.exports.EventStatus = {
+const EventStatus = {
     /** The event was not sent and will no longer be retried. */
     NOT_SENT: "not_sent",
 
@@ -49,6 +49,7 @@ module.exports.EventStatus = {
     /** The event was cancelled before it was successfully sent. */
     CANCELLED: "cancelled",
 };
+module.exports.EventStatus = EventStatus;
 
 const interns = {};
 function intern(str) {
@@ -753,6 +754,15 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return Boolean(this.getUnsigned().redacted_because);
     },
 
+    /**
+     * Check if this event is a redaction of another event
+     *
+     * @return {boolean} True if this event is a redaction
+     */
+    isRedaction: function() {
+        return this.getType() === "m.room.redaction";
+    },
+
     /**
      * Get the push actions, if known, for this event
      *
@@ -776,9 +786,26 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @param {Object} event the object to assign to the `event` property
      */
     handleRemoteEcho: function(event) {
+        const oldUnsigned = this.getUnsigned();
+        const oldId = this.getId();
         this.event = event;
+        // if this event was redacted before it was sent, it's locally marked as redacted.
+        // At this point, we've received the remote echo for the event, but not yet for
+        // the redaction that we are sending ourselves. Preserve the locally redacted
+        // state by copying over redacted_because so we don't get a flash of
+        // redacted, not-redacted, redacted as remote echos come in
+        if (oldUnsigned.redacted_because) {
+            if (!this.event.unsigned) {
+                this.event.unsigned = {};
+            }
+            this.event.unsigned.redacted_because = oldUnsigned.redacted_because;
+        }
         // successfully sent.
         this.setStatus(null);
+        if (this.getId() !== oldId) {
+            // emit the event if it changed
+            this.emit("Event.localEventIdReplaced", this);
+        }
     },
 
     /**
@@ -801,6 +828,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         this.emit("Event.status", this, status);
     },
 
+    replaceLocalEventId(eventId) {
+        this.event.event_id = eventId;
+        this.emit("Event.localEventIdReplaced", this);
+    },
+
     /**
      * Get whether the event is a relation event, and of a given type if
      * `relType` is passed in.
@@ -846,16 +878,24 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     /**
-     * Returns the status of the event, or the replacing event in case `makeReplace` has been called.
+     * Returns the status of any associated edit or redaction
+     * (not for reactions/annotations as their local echo doesn't affect the orignal event),
+     * or else the status of the event.
      *
      * @return {EventStatus}
      */
-    replacementOrOwnStatus() {
+    getAssociatedStatus() {
         if (this._replacingEvent) {
             return this._replacingEvent.status;
-        } else {
-            return this.status;
+        } else if (this._locallyRedacted) {
+            const unsigned = this.event.unsigned;
+            const redactedBecause = unsigned && unsigned.redacted_because;
+            const redactionId = redactedBecause && redactedBecause.event_id;
+            if (redactionId && redactionId.startsWith("~")) {
+                return EventStatus.SENDING;
+            }
         }
+        return this.status;
     },
 
     /**
@@ -876,6 +916,46 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return this._replacingEvent;
     },
 
+    /**
+     * For relations and redactions, returns the event_id this event is referring to.
+     *
+     * @return {string?}
+     */
+    getAssociatedId() {
+        const relation = this.getRelation();
+        if (relation) {
+            return relation.event_id;
+        } else if (this.isRedaction()) {
+            return this.event.redacts;
+        }
+    },
+
+    /**
+     * Checks if this event is associated with another event. See `getAssociatedId`.
+     *
+     * @return {bool}
+     */
+    hasAssocation() {
+        return !!this.getAssociatedId();
+    },
+
+    /**
+     * Update the related id with a new one.
+     *
+     * Used to replace a local id with remote one before sending
+     * an event with a related id.
+     *
+     * @param {string} eventId the new event id
+     */
+    updateAssociatedId(eventId) {
+        const relation = this.getRelation();
+        if (relation) {
+            relation.event_id = eventId;
+        } else if (this.isRedaction()) {
+            this.event.redacts = eventId;
+        }
+    },
+
     /**
      * Summarise the event as JSON for debugging. If encrypted, include both the
      * decrypted and encrypted view of the event. This is named `toJSON` for use

src/models/relations.js

@@ -242,12 +242,7 @@ export default class Relations extends EventEmitter {
 
         redactedEvent.removeListener("Event.beforeRedaction", this._onBeforeRedaction);
 
-        // Dispatch a redaction event on this collection. `setTimeout` is used
-        // to wait until the next event loop iteration by which time the event
-        // has actually been marked as redacted.
-        setTimeout(() => {
-            this.emit("Relations.redaction");
-        }, 0);
+        this.emit("Relations.redaction");
     }
 
     /**

src/models/room.js

@@ -1032,7 +1032,7 @@ Room.prototype.removeFilteredTimelineSet = function(filter) {
  * @private
  */
 Room.prototype._addLiveEvent = function(event, duplicateStrategy) {
-    if (event.getType() === "m.room.redaction") {
+    if (event.isRedaction()) {
         const redactId = event.event.redacts;
 
         // if we know about this event, redact its contents now.
@@ -1141,9 +1141,13 @@ Room.prototype.addPendingEvent = function(event, txnId) {
             this._aggregateNonLiveRelation(event);
         }
 
-        if (event.getType() === "m.room.redaction") {
+        if (event.isRedaction()) {
             const redactId = event.event.redacts;
-            const redactedEvent = this.getUnfilteredTimelineSet().findEventById(redactId);
+            let redactedEvent = this._pendingEventList &&
+                this._pendingEventList.find(e => e.getId() === redactId);
+            if (!redactedEvent) {
+                redactedEvent = this.getUnfilteredTimelineSet().findEventById(redactId);
+            }
             if (redactedEvent) {
                 redactedEvent.markLocallyRedacted(event);
                 this.emit("Room.redaction", event, this);
@@ -1211,7 +1215,7 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
     const oldStatus = localEvent.status;
 
     // no longer pending
-    delete this._txnToEvent[remoteEvent.transaction_id];
+    delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];
 
     // if it's in the pending list, remove it
     if (this._pendingEventList) {
@@ -1315,7 +1319,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
 
     if (newStatus == EventStatus.SENT) {
         // update the event id
-        event.event.event_id = newEventId;
+        event.replaceLocalEventId(newEventId);
 
         // if the event was already in the timeline (which will be the case if
         // opts.pendingEventOrdering==chronological), we need to update the
@@ -1329,7 +1333,7 @@ Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
             const idx = this._pendingEventList.findIndex(ev => ev.getId() === oldEventId);
             if (idx !== -1) {
                 const [removedEvent] = this._pendingEventList.splice(idx, 1);
-                if (removedEvent.getType() === "m.room.redaction") {
+                if (removedEvent.isRedaction()) {
                     this._revertRedactionLocalEcho(removedEvent);
                 }
             }
@@ -1435,7 +1439,7 @@ Room.prototype.removeEvent = function(eventId) {
     for (let i = 0; i < this._timelineSets.length; i++) {
         const removed = this._timelineSets[i].removeEvent(eventId);
         if (removed) {
-            if (removed.getType() === "m.room.redaction") {
+            if (removed.isRedaction()) {
                 this._revertRedactionLocalEcho(removed);
             }
             removedAny = true;

src/scheduler.js

@@ -177,7 +177,8 @@ MatrixScheduler.RETRY_BACKOFF_RATELIMIT = function(event, attempts, err) {
  * @see module:scheduler~queueAlgorithm
  */
 MatrixScheduler.QUEUE_MESSAGES = function(event) {
-    if (event.getType() === "m.room.message") {
+    // enqueue messages or events that associate with another event (redactions and relations)
+    if (event.getType() === "m.room.message" || event.hasAssocation()) {
         // put these events in the 'message' queue.
         return "message";
     }
@@ -220,7 +221,14 @@ function _processQueue(scheduler, queueName) {
     );
     // fire the process function and if it resolves, resolve the deferred. Else
     // invoke the retry algorithm.
-    scheduler._procFn(obj.event).done(function(res) {
+
+    // First wait for a resolved promise, so the resolve handlers for
+    // the deferred of the previously sent event can run.
+    // This way enqueued relations/redactions to enqueued events can receive
+    // the remove id of their target before being sent.
+    Promise.resolve().then(() => {
+        return scheduler._procFn(obj.event);
+    }).then(function(res) {
         // remove this from the queue
         _removeNextEvent(scheduler, queueName);
         debuglog("Queue '%s' sent event %s", queueName, obj.event.getId());

src/store/indexeddb.js

@@ -198,11 +198,13 @@ IndexedDBStore.prototype.wantsSave = function() {
 
 /**
  * Possibly write data to the database.
+ *
+ * @param {bool} force True to force a save to happen
  * @return {Promise} Promise resolves after the write completes
  *     (or immediately if no write is performed)
  */
-IndexedDBStore.prototype.save = function() {
-    if (this.wantsSave()) {
+IndexedDBStore.prototype.save = function(force) {
+    if (force || this.wantsSave()) {
         return this._reallySave();
     }
     return Promise.resolve();

src/store/memory.js

@@ -335,8 +335,10 @@ module.exports.MemoryStore.prototype = {
 
     /**
      * Save does nothing as there is no backing data store.
+     * @param {bool} force True to force a save (but the memory
+     *     store still can't save anything)
      */
-    save: function() {},
+    save: function(force) {},
 
     /**
      * Startup does nothing as this store doesn't require starting up.