v2.1.0-rc.1

Handle self read receipts for fixing e2e notification counts

CHANGELOG.md

@@ -1,3 +1,32 @@
+Changes in [2.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.1.0-rc.1) (2019-07-03)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1...v2.1.0-rc.1)
+
+ * Handle self read receipts for fixing e2e notification counts
+   [\#974](https://github.com/matrix-org/matrix-js-sdk/pull/974)
+ * Add redacts field to event.toJSON
+   [\#973](https://github.com/matrix-org/matrix-js-sdk/pull/973)
+ * Handle associated event send failures
+   [\#972](https://github.com/matrix-org/matrix-js-sdk/pull/972)
+ * Remove irrelevant debug line from timeline handling
+   [\#971](https://github.com/matrix-org/matrix-js-sdk/pull/971)
+ * Handle relations in encrypted rooms
+   [\#969](https://github.com/matrix-org/matrix-js-sdk/pull/969)
+ * Relations endpoint support
+   [\#967](https://github.com/matrix-org/matrix-js-sdk/pull/967)
+ * Disable event encryption for reactions
+   [\#968](https://github.com/matrix-org/matrix-js-sdk/pull/968)
+ * Change the known safe room version to version 4
+   [\#966](https://github.com/matrix-org/matrix-js-sdk/pull/966)
+ * Check for lazy-loading support in the spec versions instead
+   [\#965](https://github.com/matrix-org/matrix-js-sdk/pull/965)
+ * Use camelCase instead of underscore
+   [\#963](https://github.com/matrix-org/matrix-js-sdk/pull/963)
+ * Time out verification attempts after 10 minutes of inactivity
+   [\#961](https://github.com/matrix-org/matrix-js-sdk/pull/961)
+ * Don't handle key verification requests which are immediately cancelled
+   [\#962](https://github.com/matrix-org/matrix-js-sdk/pull/962)
+
 Changes in [2.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v2.0.1) (2019-06-19)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v2.0.1-rc.2...v2.0.1)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "2.0.1",
+  "version": "2.1.0-rc.1",
   "description": "Matrix Client-Server SDK for Javascript",
   "main": "index.js",
   "scripts": {

src/ReEmitter.js

@@ -34,12 +34,18 @@ export default class Reemitter {
     }
 
     reEmit(source, eventNames) {
+        // We include the source as the last argument for event handlers which may need it,
+        // such as read receipt listeners on the client class which won't have the context
+        // of the room.
+        const forSource = (handler, ...args) => {
+            handler(...args, source);
+        };
         for (const eventName of eventNames) {
             if (this.boundHandlers[eventName] === undefined) {
                 this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
             }
-            const boundHandler = this.boundHandlers[eventName];
 
+            const boundHandler = forSource.bind(this, this.boundHandlers[eventName]);
             source.on(eventName, boundHandler);
         }
     }

src/base-apis.js

@@ -438,6 +438,35 @@ MatrixBaseApis.prototype.createRoom = function(options, callback) {
         callback, "POST", "/createRoom", undefined, options,
     );
 };
+/**
+ * Fetches relations for a given event
+ * @param {string} roomId the room of the event
+ * @param {string} eventId the id of the event
+ * @param {string} relationType the rel_type of the relations requested
+ * @param {string} eventType the event type of the relations requested
+ * @param {Object} opts options with optional values for the request.
+ * @param {Object} opts.from the pagination token returned from a previous request as `next_batch` to return following relations.
+ * @return {Object} the response, with chunk and next_batch.
+ */
+MatrixBaseApis.prototype.fetchRelations =
+    async function(roomId, eventId, relationType, eventType, opts) {
+    const queryParams = {};
+    if (opts.from) {
+        queryParams.from = opts.from;
+    }
+    const queryString = utils.encodeParams(queryParams);
+    const path = utils.encodeUri(
+        "/rooms/$roomId/relations/$eventId/$relationType/$eventType?" + queryString, {
+            $roomId: roomId,
+            $eventId: eventId,
+            $relationType: relationType,
+            $eventType: eventType,
+        });
+    const response = await this._http.authedRequestWithPrefix(
+        undefined, "GET", path, null, null, httpApi.PREFIX_UNSTABLE,
+    );
+    return response;
+};
 
 /**
  * @param {string} roomId

src/client.js

@@ -278,6 +278,46 @@ function MatrixClient(opts) {
             }
         }
     });
+
+    // Like above, we have to listen for read receipts from ourselves in order to
+    // correctly handle notification counts on encrypted rooms.
+    // This fixes https://github.com/vector-im/riot-web/issues/9421
+    this.on("Room.receipt", (event, room) => {
+        if (room && this.isRoomEncrypted(room.roomId)) {
+            // Figure out if we've read something or if it's just informational
+            const content = event.getContent();
+            const isSelf = Object.keys(content).filter(eid => {
+                return Object.keys(content[eid]['m.read']).includes(this.getUserId());
+            }).length > 0;
+
+            if (!isSelf) return;
+
+            // Work backwards to determine how many events are unread. We also set
+            // a limit for how back we'll look to avoid spinning CPU for too long.
+            // If we hit the limit, we assume the count is unchanged.
+            const maxHistory = 20;
+            const events = room.getLiveTimeline().getEvents();
+
+            let highlightCount = 0;
+
+            for (let i = events.length - 1; i >= 0; i--) {
+                if (i === events.length - maxHistory) return; // limit reached
+
+                const event = events[i];
+
+                if (room.hasUserReadEvent(this.getUserId(), event.getId())) {
+                    // If the user has read the event, then the counting is done.
+                    break;
+                }
+
+                highlightCount += event.getPushActions().tweaks.highlight ? 1 : 0;
+            }
+
+            // Note: we don't need to handle 'total' notifications because the counts
+            // will come from the server.
+            room.setUnreadNotificationCount("highlight", highlightCount);
+        }
+    });
 }
 utils.inherits(MatrixClient, EventEmitter);
 utils.extend(MatrixClient.prototype, MatrixBaseApis.prototype);
@@ -1863,6 +1903,20 @@ function _encryptEventIfNeeded(client, event, room) {
         return null;
     }
 
+    if (event.getType() === "m.reaction") {
+        // For reactions, there is a very little gained by encrypting the entire
+        // event, as relation data is already kept in the clear. Event
+        // encryption for a reaction effectively only obscures the event type,
+        // but the purpose is still obvious from the relation data, so nothing
+        // is really gained. It also causes quite a few problems, such as:
+        //   * triggers notifications via default push rules
+        //   * prevents server-side bundling for reactions
+        // The reaction key / content / emoji value does warrant encrypting, but
+        // this will be handled separately by encrypting just this value.
+        // See https://github.com/matrix-org/matrix-doc/pull/1849#pullrequestreview-248763642
+        return null;
+    }
+
     if (!client._crypto) {
         throw new Error(
             "This room is configured to use encryption, but your client does " +
@@ -1872,6 +1926,21 @@ function _encryptEventIfNeeded(client, event, room) {
 
     return client._crypto.encryptEvent(event, room);
 }
+/**
+ * Returns the eventType that should be used taking encryption into account
+ * for a given eventType.
+ * @param {MatrixClient} client the client
+ * @param {string} roomId the room for the events `eventType` relates to
+ * @param {string} eventType the event type
+ * @return {string} the event type taking encryption into account
+ */
+function _getEncryptedIfNeededEventType(client, roomId, eventType) {
+    if (eventType === "m.reaction") {
+        return eventType;
+    }
+    const isEncrypted = client.isRoomEncrypted(roomId);
+    return isEncrypted ? "m.room.encrypted" : eventType;
+}
 
 function _updatePendingEventStatus(room, event, newStatus) {
     if (room) {
@@ -3925,9 +3994,13 @@ MatrixClient.prototype.doesServerSupportLazyLoading = async function() {
                 prefix: '',
             },
         );
+
+        const versions = response["versions"];
         const unstableFeatures = response["unstable_features"];
+
         this._serverSupportsLazyLoading =
-            unstableFeatures && unstableFeatures["m.lazy_load_members"];
+            (versions && versions.includes("r0.5.0"))
+            || (unstableFeatures && unstableFeatures["m.lazy_load_members"]);
     }
     return this._serverSupportsLazyLoading;
 };
@@ -3961,6 +4034,41 @@ MatrixClient.prototype.getCanResetTimelineCallback = function() {
     return this._canResetTimelineCallback;
 };
 
+/**
+ * Returns relations for a given event. Handles encryption transparently,
+ * with the caveat that the amount of events returned might be 0, even though you get a nextBatch.
+ * When the returned promise resolves, all messages should have finished trying to decrypt.
+ * @param {string} roomId the room of the event
+ * @param {string} eventId the id of the event
+ * @param {string} relationType the rel_type of the relations requested
+ * @param {string} eventType the event type of the relations requested
+ * @param {Object} opts options with optional values for the request.
+ * @param {Object} opts.from the pagination token returned from a previous request as `nextBatch` to return following relations.
+ * @return {Object} an object with `events` as `MatrixEvent[]` and optionally `nextBatch` if more relations are available.
+ */
+MatrixClient.prototype.relations =
+async function(roomId, eventId, relationType, eventType, opts = {}) {
+    const fetchedEventType = _getEncryptedIfNeededEventType(this, roomId, eventType);
+    const result = await this.fetchRelations(
+        roomId,
+        eventId,
+        relationType,
+        fetchedEventType,
+        opts);
+
+    let events = result.chunk.map(this.getEventMapper());
+    if (fetchedEventType === "m.room.encrypted") {
+        await Promise.all(events.map(e => {
+            return new Promise(resolve => e.once("Event.decrypted", resolve));
+        }));
+        events = events.filter(e => e.getType() === eventType);
+    }
+    return {
+        events,
+        nextBatch: result.next_batch,
+    };
+};
+
 function setupCallEventHandler(client) {
     const candidatesByCall = {
         // callId: [Candidate]

src/crypto/index.js

@@ -1648,6 +1648,11 @@ Crypto.prototype._onRoomKeyEvent = function(event) {
  * @param {module:models/event.MatrixEvent} event verification request event
  */
 Crypto.prototype._onKeyVerificationRequest = function(event) {
+    if (event.isCancelled()) {
+        logger.warn("Ignoring flagged verification request from " + event.getSender());
+        return;
+    }
+
     const content = event.getContent();
     if (!("from_device" in content) || typeof content.from_device !== "string"
         || !("transaction_id" in content) || typeof content.from_device !== "string"
@@ -1764,6 +1769,11 @@ Crypto.prototype._onKeyVerificationRequest = function(event) {
  * @param {module:models/event.MatrixEvent} event verification start event
  */
 Crypto.prototype._onKeyVerificationStart = function(event) {
+    if (event.isCancelled()) {
+        logger.warn("Ignoring flagged verification start from " + event.getSender());
+        return;
+    }
+
     const sender = event.getSender();
     const content = event.getContent();
     const transactionId = content.transaction_id;

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -702,7 +702,7 @@ function promiseifyTxn(txn) {
             if (txn._mx_abortexception !== undefined) {
                 reject(txn._mx_abortexception);
             } else {
-                localStorage.log("Error performing indexeddb txn", event);
+                logger.log("Error performing indexeddb txn", event);
                 reject(event.target.error);
             }
         };

src/crypto/verification/Base.js

@@ -22,6 +22,9 @@ limitations under the License.
 import {MatrixEvent} from '../../models/event';
 import {EventEmitter} from 'events';
 import logger from '../../logger';
+import {newTimeoutError} from "./Error";
+
+const timeoutException = new Error("Verification timed out");
 
 export default class VerificationBase extends EventEmitter {
     /**
@@ -60,9 +63,27 @@ export default class VerificationBase extends EventEmitter {
         this.transactionId = transactionId;
         this.startEvent = startEvent;
         this.request = request;
+        this.cancelled = false;
         this._parent = parent;
         this._done = false;
         this._promise = null;
+        this._transactionTimeoutTimer = null;
+
+        // At this point, the verification request was received so start the timeout timer.
+        this._resetTimer();
+    }
+
+    _resetTimer() {
+        console.log("Refreshing/starting the verification transaction timeout timer");
+        if (this._transactionTimeoutTimer !== null) {
+            clearTimeout(this._transactionTimeoutTimer);
+        }
+        setTimeout(() => {
+           if (!this._done && !this.cancelled) {
+               console.log("Triggering verification timeout");
+               this.cancel(timeoutException);
+           }
+        }, 10 * 60 * 1000); // 10 minutes
     }
 
     _sendToDevice(type, content) {
@@ -92,6 +113,7 @@ export default class VerificationBase extends EventEmitter {
         } else if (e.getType() === this._expectedEvent) {
             this._expectedEvent = undefined;
             this._rejectEvent = undefined;
+            this._resetTimer();
             this._resolveEvent(e);
         } else {
             this._expectedEvent = undefined;
@@ -116,10 +138,14 @@ export default class VerificationBase extends EventEmitter {
 
     cancel(e) {
         if (!this._done) {
+            this.cancelled = true;
             if (this.userId && this.deviceId && this.transactionId) {
                 // send a cancellation to the other user (if it wasn't
                 // cancelled by the other user)
-                if (e instanceof MatrixEvent) {
+                if (e === timeoutException) {
+                    const timeoutEvent = newTimeoutError();
+                    this._sendToDevice(timeoutEvent.getType(), timeoutEvent.getContent());
+                } else if (e instanceof MatrixEvent) {
                     const sender = e.getSender();
                     if (sender !== this.userId) {
                         const content = e.getContent();
@@ -146,7 +172,9 @@ export default class VerificationBase extends EventEmitter {
                 }
             }
             if (this._promise !== null) {
-                this._reject(e);
+                // when we cancel without a promise, we end up with a promise
+                // but no reject function. If cancel is called again, we'd error.
+                if (this._reject) this._reject(e);
             } else {
                 this._promise = Promise.reject(e);
             }
@@ -177,6 +205,7 @@ export default class VerificationBase extends EventEmitter {
         });
         if (this._doVerification && !this._started) {
             this._started = true;
+            this._resetTimer(); // restart the timeout
             Promise.resolve(this._doVerification())
                 .then(this.done.bind(this), this.cancel.bind(this));
         }

src/models/event-timeline-set.js

@@ -438,7 +438,6 @@ EventTimelineSet.prototype.addEventsToTimeline = function(events, toStartOfTimel
         if (backwardsIsLive || forwardsIsLive) {
             // The live timeline should never be spliced into a non-live position.
             // We use independent logging to better discover the problem at a glance.
-            logger.warn({backwardsIsLive, forwardsIsLive}); // debugging
             if (backwardsIsLive) {
                 logger.warn(
                     "Refusing to set a preceding existingTimeLine on our " +

src/models/event.js

@@ -125,7 +125,8 @@ module.exports.MatrixEvent = function MatrixEvent(
     this.forwardLooking = true;
     this._pushActions = null;
     this._replacingEvent = null;
-    this._locallyRedacted = false;
+    this._localRedactionEvent = null;
+    this._isCancelled = false;
 
     this._clearEvent = {};
 
@@ -230,7 +231,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Object} The event content JSON, or an empty object.
      */
     getOriginalContent: function() {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return {};
         }
         return this._clearEvent.content || this.event.content || {};
@@ -244,7 +245,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @return {Object} The event content JSON, or an empty object.
      */
     getContent: function() {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return {};
         } else if (this._replacingEvent) {
             return this._replacingEvent.getContent()["m.new_content"] || {};
@@ -674,20 +675,20 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     },
 
     unmarkLocallyRedacted: function() {
-        const value = this._locallyRedacted;
-        this._locallyRedacted = false;
+        const value = this._localRedactionEvent;
+        this._localRedactionEvent = null;
         if (this.event.unsigned) {
             this.event.unsigned.redacted_because = null;
         }
-        return value;
+        return !!value;
     },
 
     markLocallyRedacted: function(redactionEvent) {
-        if (this._locallyRedacted) {
+        if (this._localRedactionEvent) {
             return;
         }
         this.emit("Event.beforeRedaction", this, redactionEvent);
-        this._locallyRedacted = true;
+        this._localRedactionEvent = redactionEvent;
         if (!this.event.unsigned) {
             this.event.unsigned = {};
         }
@@ -707,7 +708,7 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             throw new Error("invalid redaction_event in makeRedacted");
         }
 
-        this._locallyRedacted = false;
+        this._localRedactionEvent = null;
 
         this.emit("Event.beforeRedaction", this, redaction_event);
 
@@ -868,7 +869,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
      * @param {MatrixEvent?} newEvent the event with the replacing content, if any.
      */
     makeReplaced(newEvent) {
-        if (this.isRedacted()) {
+        // don't allow redacted events to be replaced.
+        // if newEvent is null we allow to go through though,
+        // as with local redaction, the replacing event might get
+        // cancelled, which should be reflected on the target event.
+        if (this.isRedacted() && newEvent) {
             return;
         }
         if (this._replacingEvent !== newEvent) {
@@ -887,13 +892,8 @@ utils.extend(module.exports.MatrixEvent.prototype, {
     getAssociatedStatus() {
         if (this._replacingEvent) {
             return this._replacingEvent.status;
-        } else if (this._locallyRedacted) {
-            const unsigned = this.event.unsigned;
-            const redactedBecause = unsigned && unsigned.redacted_because;
-            const redactionId = redactedBecause && redactedBecause.event_id;
-            if (redactionId && redactionId.startsWith("~")) {
-                return EventStatus.SENDING;
-            }
+        } else if (this._localRedactionEvent) {
+            return this._localRedactionEvent.status;
         }
         return this.status;
     },
@@ -909,6 +909,8 @@ utils.extend(module.exports.MatrixEvent.prototype, {
 
     /**
      * Returns the event replacing the content of this event, if any.
+     * Replacements are aggregated on the server, so this would only
+     * return an event in case it came down the sync, or for local echo of edits.
      *
      * @return {MatrixEvent?}
      */
@@ -916,6 +918,14 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         return this._replacingEvent;
     },
 
+    /**
+     * Returns the event that wants to redact this event, but hasn't been sent yet.
+     * @return {MatrixEvent} the event
+     */
+    localRedactionEvent() {
+        return this._localRedactionEvent;
+    },
+
     /**
      * For relations and redactions, returns the event_id this event is referring to.
      *
@@ -956,6 +966,25 @@ utils.extend(module.exports.MatrixEvent.prototype, {
         }
     },
 
+    /**
+     * Flags an event as cancelled due to future conditions. For example, a verification
+     * request event in the same sync transaction may be flagged as cancelled to warn
+     * listeners that a cancellation event is coming down the same pipe shortly.
+     * @param {boolean} cancelled Whether the event is to be cancelled or not.
+     */
+    flagCancelled(cancelled = true) {
+        this._isCancelled = cancelled;
+    },
+
+    /**
+     * Gets whether or not the event is flagged as cancelled. See flagCancelled() for
+     * more information.
+     * @returns {boolean} True if the event is cancelled, false otherwise.
+     */
+    isCancelled() {
+        return this._isCancelled;
+    },
+
     /**
      * Summarise the event as JSON for debugging. If encrypted, include both the
      * decrypted and encrypted view of the event. This is named `toJSON` for use
@@ -975,6 +1004,11 @@ utils.extend(module.exports.MatrixEvent.prototype, {
             room_id: this.getRoomId(),
         };
 
+        // if this is a redaction then attach the redacts key
+        if (this.isRedaction()) {
+            event.redacts = this.event.redacts;
+        }
+
         if (!this.isEncrypted()) {
             return event;
         }

src/models/room.js

@@ -38,7 +38,7 @@ import ReEmitter from '../ReEmitter';
 // room versions which are considered okay for people to run without being asked
 // to upgrade (ie: "stable"). Eventually, we should remove these when all homeservers
 // return an m.room_versions capability.
-const KNOWN_SAFE_ROOM_VERSION = '1';
+const KNOWN_SAFE_ROOM_VERSION = '4';
 const SAFE_ROOM_VERSIONS = ['1', '2', '3', '4'];
 
 function synthesizeReceipt(userId, event, receiptType) {
@@ -797,20 +797,20 @@ Room.prototype.getAvatarUrl = function(baseUrl, width, height, resizeMethod,
  * @return {array} The room's alias as an array of strings
  */
 Room.prototype.getAliases = function() {
-    const alias_strings = [];
+    const aliasStrings = [];
 
-    const alias_events = this.currentState.getStateEvents("m.room.aliases");
-    if (alias_events) {
-        for (let i = 0; i < alias_events.length; ++i) {
-            const alias_event = alias_events[i];
-            if (utils.isArray(alias_event.getContent().aliases)) {
+    const aliasEvents = this.currentState.getStateEvents("m.room.aliases");
+    if (aliasEvents) {
+        for (let i = 0; i < aliasEvents.length; ++i) {
+            const aliasEvent = aliasEvents[i];
+            if (utils.isArray(aliasEvent.getContent().aliases)) {
                 Array.prototype.push.apply(
-                    alias_strings, alias_event.getContent().aliases,
+                    aliasStrings, aliasEvent.getContent().aliases,
                 );
             }
         }
     }
-    return alias_strings;
+    return aliasStrings;
 };
 
 /**
@@ -1419,11 +1419,11 @@ Room.prototype.addLiveEvents = function(events, duplicateStrategy) {
 
 /**
  * Removes events from this room.
- * @param {String[]} event_ids A list of event_ids to remove.
+ * @param {String[]} eventIds A list of eventIds to remove.
  */
-Room.prototype.removeEvents = function(event_ids) {
-    for (let i = 0; i < event_ids.length; ++i) {
-        this.removeEvent(event_ids[i]);
+Room.prototype.removeEvents = function(eventIds) {
+    for (let i = 0; i < eventIds.length; ++i) {
+        this.removeEvent(eventIds[i]);
     }
 };
 

src/sync.js

@@ -1043,8 +1043,26 @@ SyncApi.prototype._processSyncResponse = async function(
     if (data.to_device && utils.isArray(data.to_device.events) &&
         data.to_device.events.length > 0
        ) {
+        const cancelledKeyVerificationTxns = [];
         data.to_device.events
             .map(client.getEventMapper())
+            .map((toDeviceEvent) => { // map is a cheap inline forEach
+                // We want to flag m.key.verification.start events as cancelled
+                // if there's an accompanying m.key.verification.cancel event, so
+                // we pull out the transaction IDs from the cancellation events
+                // so we can flag the verification events as cancelled in the loop
+                // below.
+                if (toDeviceEvent.getType() === "m.key.verification.cancel") {
+                    const txnId = toDeviceEvent.getContent()['transaction_id'];
+                    if (txnId) {
+                        cancelledKeyVerificationTxns.push(txnId);
+                    }
+                }
+
+                // as mentioned above, .map is a cheap inline forEach, so return
+                // the unmodified event.
+                return toDeviceEvent;
+            })
             .forEach(
                 function(toDeviceEvent) {
                     const content = toDeviceEvent.getContent();
@@ -1060,6 +1078,14 @@ SyncApi.prototype._processSyncResponse = async function(
                         return;
                     }
 
+                    if (toDeviceEvent.getType() === "m.key.verification.start"
+                        || toDeviceEvent.getType() === "m.key.verification.request") {
+                        const txnId = content['transaction_id'];
+                        if (cancelledKeyVerificationTxns.includes(txnId)) {
+                            toDeviceEvent.flagCancelled();
+                        }
+                    }
+
                     client.emit("toDeviceEvent", toDeviceEvent);
                 },
             );