From 2b659656cc74dff9836483400d871178c1abd85a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 8 Dec 2015 17:41:09 +0000 Subject: [PATCH 01/32] Move alllll the sync code to sync.js - still more to do (in FIXME XXX) --- lib/client.js | 489 ++------------------------------------------------ lib/sync.js | 415 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 426 insertions(+), 478 deletions(-) create mode 100644 lib/sync.js diff --git a/lib/client.js b/lib/client.js index 1ec64be36..e9c77469a 100644 --- a/lib/client.js +++ b/lib/client.js @@ -13,12 +13,11 @@ var httpApi = require("./http-api"); var MatrixEvent = require("./models/event").MatrixEvent; var EventStatus = require("./models/event").EventStatus; var StubStore = require("./store/stub"); -var Room = require("./models/room"); -var User = require("./models/user"); var webRtcCall = require("./webrtc/call"); var utils = require("./utils"); var contentRepo = require("./content-repo"); var Filter = require("./filter"); +var SyncApi = require("./sync"); var SCROLLBACK_DELAY_MS = 3000; var CRYPTO_ENABLED = false; @@ -622,9 +621,9 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) { this._http.authedRequest(undefined, "POST", path, undefined, {}).then( function(res) { var roomId = res.room_id; - var room = createNewRoom(self, roomId); + var room = createNewRoom(self, roomId); // XXX FIXME TODO if (opts.syncRoom) { - return _syncRoom(self, room); + return _syncRoom(self, room); // XXX FIXME TODO } return q(room); }, function(err) { @@ -2174,136 +2173,12 @@ MatrixClient.prototype.isLoggedIn = function() { /** * This is an internal method. * @param {MatrixClient} client - * @param {integer} historyLen - * @param {integer} includeArchived - * @param {integer} attempt */ -function doInitialSync(client, historyLen, includeArchived, attempt) { - attempt = attempt || 1; - var qps = { limit: historyLen }; - if (includeArchived) { - qps.archived = true; - } - if (client._guestRooms && client._isGuest) { - console.log(client._guestRooms); - qps.room_id = JSON.stringify(client._guestRooms); - } - client._http.authedRequest( - undefined, "GET", "/initialSync", qps - ).done(function(data) { - var i, j; - // intercept the results and put them into our store - if (!(client.store instanceof StubStore)) { - utils.forEach( - utils.map(data.presence, _PojoToMatrixEventMapper(client)), - function(e) { - var user = createNewUser(client, e.getContent().user_id); - user.setPresenceEvent(e); - client.store.storeUser(user); - }); - - // group receipts by room ID. - var receiptsByRoom = {}; - data.receipts = data.receipts || []; - utils.forEach(data.receipts.map(_PojoToMatrixEventMapper(client)), - function(receiptEvent) { - if (!receiptsByRoom[receiptEvent.getRoomId()]) { - receiptsByRoom[receiptEvent.getRoomId()] = []; - } - receiptsByRoom[receiptEvent.getRoomId()].push(receiptEvent); - } - ); - - for (i = 0; i < data.rooms.length; i++) { - var room = createNewRoom(client, data.rooms[i].room_id); - if (!data.rooms[i].state) { - data.rooms[i].state = []; - } - if (data.rooms[i].membership === "invite") { - var inviteEvent = data.rooms[i].invite; - if (!inviteEvent) { - // fallback for servers which don't serve the invite key yet - inviteEvent = { - event_id: "$fake_" + room.roomId, - content: { - membership: "invite" - }, - state_key: client.credentials.userId, - user_id: data.rooms[i].inviter, - room_id: room.roomId, - type: "m.room.member" - }; - } - data.rooms[i].state.push(inviteEvent); - } - - _processRoomEvents( - client, room, data.rooms[i].state, data.rooms[i].messages - ); - - var receipts = receiptsByRoom[room.roomId] || []; - for (j = 0; j < receipts.length; j++) { - room.addReceipt(receipts[j]); - } - - var privateUserData = data.rooms[i].account_data || []; - var privateUserDataEvents = - utils.map(privateUserData, _PojoToMatrixEventMapper(client)); - for (j = 0; j < privateUserDataEvents.length; j++) { - var event = privateUserDataEvents[j]; - if (event.getType() === "m.tag") { - room.addTags(event); - } - // XXX: unhandled private user data event - we should probably - // put it somewhere useful once the API has settled - } - - // cache the name/summary/etc prior to storage since we don't - // know how the store will serialise the Room. - room.recalculate(client.credentials.userId); - - client.store.storeRoom(room); - client.emit("Room", room); - } - } - - if (data) { - client.store.setSyncToken(data.end); - var events = []; - for (i = 0; i < data.presence.length; i++) { - events.push(new MatrixEvent(data.presence[i])); - } - for (i = 0; i < data.rooms.length; i++) { - if (data.rooms[i].state) { - for (j = 0; j < data.rooms[i].state.length; j++) { - events.push(new MatrixEvent(data.rooms[i].state[j])); - } - } - if (data.rooms[i].messages) { - for (j = 0; j < data.rooms[i].messages.chunk.length; j++) { - events.push( - new MatrixEvent(data.rooms[i].messages.chunk[j]) - ); - } - } - } - utils.forEach(events, function(e) { - client.emit("event", e); - }); - } - - client.clientRunning = true; - updateSyncState(client, "PREPARED"); - // assume success until we fail which may be 30+ secs - updateSyncState(client, "SYNCING"); - _pollForEvents(client); - }, function(err) { - console.error("/initialSync error (%s attempts): %s", attempt, err); - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - doInitialSync(client, historyLen, includeArchived, attempt); - }); - updateSyncState(client, "ERROR", { error: err }); +function doSync(client) { + var syncApi = new SyncApi(client); + syncApi.sync({ + historyLen: client._config.initialSyncLimit, + includeArchived: client._config.includeArchivedRooms }); } @@ -2352,250 +2227,12 @@ MatrixClient.prototype.startClient = function(opts) { this.uploadKeys(5); } - if (this.store.getSyncToken()) { - // resume from where we left off. - _pollForEvents(this); - return; - } - // periodically poll for turn servers if we support voip checkTurnServers(this); - prepareForSync(this); + doSync(this); }; -function prepareForSync(client, attempt) { - if (client.isGuest()) { - // no push rules for guests - doInitialSync( - client, - client._config.initialSyncLimit, - client._config.includeArchivedRooms - ); - return; - } - - attempt = attempt || 1; - // we do push rules before syncing so when we gets events down we know immediately - // whether they are bing-worthy. - client.pushRules().done(function(result) { - client.pushRules = result; - doInitialSync( - client, - client._config.initialSyncLimit, - client._config.includeArchivedRooms - ); - }, function(err) { - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - prepareForSync(client, attempt); - }); - updateSyncState(client, "ERROR", { error: err }); - }); -} - -/** - * This is an internal method. - * @param {MatrixClient} client - * @param {Number} attempt The attempt number - */ -function _pollForEvents(client, attempt) { - attempt = attempt || 1; - var self = client; - if (!client.clientRunning) { - return; - } - var timeoutMs = client._config.pollTimeout; - if (attempt > 1) { - // we think the connection is dead. If it comes back up, we won't know - // about it till /events returns. If the timeout= is high, this could - // be a long time. Set it to 1 when doing retries. - timeoutMs = 1; - } - var discardResult = false; - var timeoutObj = setTimeout(function() { - discardResult = true; - console.error("/events request timed out."); - _pollForEvents(client); - }, timeoutMs + (20 * 1000)); // 20s buffer - - var queryParams = { - from: client.store.getSyncToken(), - timeout: timeoutMs - }; - if (client._guestRooms && client._isGuest) { - queryParams.room_id = client._guestRooms; - } - - client._http.authedRequest(undefined, "GET", "/events", queryParams).done( - function(data) { - if (discardResult) { - return; - } - else { - clearTimeout(timeoutObj); - } - - if (self._syncState !== "SYNCING") { - updateSyncState(self, "SYNCING"); - } - - try { - var events = []; - if (data) { - events = utils.map(data.chunk, _PojoToMatrixEventMapper(self)); - } - if (!(self.store instanceof StubStore)) { - var roomIdsWithNewInvites = {}; - // bucket events based on room. - var i = 0; - var roomIdToEvents = {}; - for (i = 0; i < events.length; i++) { - var roomId = events[i].getRoomId(); - // possible to have no room ID e.g. for presence events. - if (roomId) { - if (!roomIdToEvents[roomId]) { - roomIdToEvents[roomId] = []; - } - roomIdToEvents[roomId].push(events[i]); - if (events[i].getType() === "m.room.member" && - events[i].getContent().membership === "invite") { - roomIdsWithNewInvites[roomId] = true; - } - } - else if (events[i].getType() === "m.presence") { - var usr = self.store.getUser(events[i].getContent().user_id); - if (usr) { - usr.setPresenceEvent(events[i]); - } - else { - usr = createNewUser(self, events[i].getContent().user_id); - usr.setPresenceEvent(events[i]); - self.store.storeUser(usr); - } - } - } - - // add events to room - var roomIds = utils.keys(roomIdToEvents); - utils.forEach(roomIds, function(roomId) { - var room = self.store.getRoom(roomId); - var isBrandNewRoom = false; - if (!room) { - room = createNewRoom(self, roomId); - isBrandNewRoom = true; - } - - var wasJoined = room.hasMembershipState( - self.credentials.userId, "join" - ); - - room.addEvents(roomIdToEvents[roomId], "replace"); - room.recalculate(self.credentials.userId); - - // store the Room for things like invite events so developers - // can update the UI - if (isBrandNewRoom) { - self.store.storeRoom(room); - self.emit("Room", room); - } - - var justJoined = room.hasMembershipState( - self.credentials.userId, "join" - ); - - if (!wasJoined && justJoined) { - // we've just transitioned into a join state for this room, - // so sync state. - _syncRoom(self, room); - } - }); - - Object.keys(roomIdsWithNewInvites).forEach(function(inviteRoomId) { - _resolveInvites(self, self.store.getRoom(inviteRoomId)); - }); - } - if (data) { - self.store.setSyncToken(data.end); - utils.forEach(events, function(e) { - self.emit("event", e); - }); - } - } - catch (e) { - console.error("Event stream error:"); - console.error(e); - } - _pollForEvents(self); - }, function(err) { - console.error("/events error: %s", JSON.stringify(err)); - if (discardResult) { - return; - } - else { - clearTimeout(timeoutObj); - } - - attempt += 1; - startSyncingRetryTimer(self, attempt, function() { - _pollForEvents(self, attempt); - }); - updateSyncState(self, "ERROR", { error: err }); - }); -} - -function _syncRoom(client, room) { - if (client._syncingRooms[room.roomId]) { - return client._syncingRooms[room.roomId]; - } - var defer = q.defer(); - client._syncingRooms[room.roomId] = defer.promise; - client.roomInitialSync(room.roomId, client._config.initialSyncLimit).done( - function(res) { - room.timeline = []; // blow away any previous messages. - _processRoomEvents(client, room, res.state, res.messages); - room.recalculate(client.credentials.userId); - client.store.storeRoom(room); - client.emit("Room", room); - defer.resolve(room); - client._syncingRooms[room.roomId] = undefined; - }, function(err) { - defer.reject(err); - client._syncingRooms[room.roomId] = undefined; - }); - return defer.promise; -} - -function _processRoomEvents(client, room, stateEventList, messageChunk) { - // "old" and "current" state are the same initially; they - // start diverging if the user paginates. - // We must deep copy otherwise membership changes in old state - // will leak through to current state! - var oldStateEvents = utils.map( - utils.deepCopy(stateEventList), _PojoToMatrixEventMapper(client) - ); - var stateEvents = utils.map(stateEventList, _PojoToMatrixEventMapper(client)); - room.oldState.setStateEvents(oldStateEvents); - room.currentState.setStateEvents(stateEvents); - - _resolveInvites(client, room); - - // add events to the timeline *after* setting the state - // events so messages use the right display names. Initial sync - // returns messages in chronological order, so we need to reverse - // it to get most recent -> oldest. We need it in that order in - // order to diverge old/current state correctly. - room.addEventsToTimeline( - utils.map( - messageChunk ? messageChunk.chunk : [], - _PojoToMatrixEventMapper(client) - ).reverse(), true - ); - if (messageChunk) { - room.oldState.paginationToken = messageChunk.start; - } -} - /** * High level helper method to stop the client from polling and allow a * clean shutdown. @@ -2605,66 +2242,6 @@ MatrixClient.prototype.stopClient = function() { // TODO: f.e. Room => self.store.storeRoom(room) ? }; - -function reEmit(reEmitEntity, emittableEntity, eventNames) { - utils.forEach(eventNames, function(eventName) { - // setup a listener on the entity (the Room, User, etc) for this event - emittableEntity.on(eventName, function() { - // take the args from the listener and reuse them, adding the - // event name to the arg list so it works with .emit() - // Transformation Example: - // listener on "foo" => function(a,b) { ... } - // Re-emit on "thing" => thing.emit("foo", a, b) - var newArgs = [eventName]; - for (var i = 0; i < arguments.length; i++) { - newArgs.push(arguments[i]); - } - reEmitEntity.emit.apply(reEmitEntity, newArgs); - }); - }); -} - -function _resolveInvites(client, room) { - if (!room || !client._config.resolveInvitesToProfiles) { - return; - } - // For each invited room member we want to give them a displayname/avatar url - // if they have one (the m.room.member invites don't contain this). - room.getMembersWithMembership("invite").forEach(function(member) { - if (member._requestedProfileInfo) { - return; - } - member._requestedProfileInfo = true; - // try to get a cached copy first. - var user = client.getUser(member.userId); - var promise; - if (user) { - promise = q({ - avatar_url: user.avatarUrl, - displayname: user.displayName - }); - } - else { - promise = client.getProfileInfo(member.userId); - } - promise.done(function(info) { - // slightly naughty by doctoring the invite event but this means all - // the code paths remain the same between invite/join display name stuff - // which is a worthy trade-off for some minor pollution. - var inviteEvent = member.events.member; - if (inviteEvent.getContent().membership !== "invite") { - // between resolving and now they have since joined, so don't clobber - return; - } - inviteEvent.getContent().avatar_url = info.avatar_url; - inviteEvent.getContent().displayname = info.displayname; - member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners - }, function(err) { - // OH WELL. - }); - }); -} - function setupCallEventHandler(client) { var candidatesByCall = { // callId: [Candidate] @@ -2820,20 +2397,6 @@ function setupCallEventHandler(client) { }); } -function startSyncingRetryTimer(client, attempt, fn) { - client._syncingRetry = {}; - client._syncingRetry.fn = fn; - client._syncingRetry.timeoutId = setTimeout(function() { - fn(); - }, retryTimeMsForAttempt(attempt)); -} - -function updateSyncState(client, newState, data) { - var old = client._syncState; - client._syncState = newState; - client.emit("sync", client._syncState, old, data); -} - function checkTurnServers(client) { if (!client._supportsVoip) { return; @@ -2861,37 +2424,6 @@ function checkTurnServers(client) { }); } -function createNewUser(client, userId) { - var user = new User(userId); - reEmit(client, user, ["User.avatarUrl", "User.displayName", "User.presence"]); - return user; -} - -function createNewRoom(client, roomId) { - var room = new Room(roomId, { - pendingEventOrdering: client._config.pendingEventOrdering - }); - reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); - - // we need to also re-emit room state and room member events, so hook it up - // to the client now. We need to add a listener for RoomState.members in - // order to hook them correctly. (TODO: find a better way?) - reEmit(client, room.currentState, [ - "RoomState.events", "RoomState.members", "RoomState.newMember" - ]); - room.currentState.on("RoomState.newMember", function(event, state, member) { - member.user = client.getUser(member.userId); - reEmit( - client, member, - [ - "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", - "RoomMember.membership" - ] - ); - }); - return room; -} - function retryTimeMsForAttempt(attempt) { // 2,4,8,16,32,64,128,128,128,... seconds // max 2^7 secs = 2.1 mins @@ -2970,7 +2502,8 @@ MatrixClient.prototype.generateClientSecret = function() { module.exports.MatrixClient = MatrixClient; /** */ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED; - +/** */ +module.exports.EventMapper = _PojoToMatrixEventMapper; // MatrixClient Event JSDocs diff --git a/lib/sync.js b/lib/sync.js new file mode 100644 index 000000000..c34e7a242 --- /dev/null +++ b/lib/sync.js @@ -0,0 +1,415 @@ +"use strict"; + +/* + * TODO: + * This class mainly serves to take all the syncing logic out of client.js and + * into a separate file. It's all very fluid, and this class gut wrenches a lot + * of MatrixClient props (e.g. _http). Given we want to support WebSockets as + * an alternative syncing API, we may want to have a proper syncing interface + * for HTTP and WS at some point. + */ + +var StubStore = require("./store/stub"); +var EventMapper = require("./client").EventMapper; +var User = require("./models/user"); +var Room = require("./models/room"); +var utils = require("./utils"); +var MatrixEvent = require("./models/event").MatrixEvent; + +function startSyncingRetryTimer(client, attempt, fn) { + client._syncingRetry = {}; + client._syncingRetry.fn = fn; + client._syncingRetry.timeoutId = setTimeout(function() { + fn(); + }, retryTimeMsForAttempt(attempt)); +} + +function updateSyncState(client, newState, data) { + var old = client._syncState; + client._syncState = newState; + client.emit("sync", client._syncState, old, data); +} + +function createNewUser(client, userId) { + var user = new User(userId); + reEmit(client, user, ["User.avatarUrl", "User.displayName", "User.presence"]); + return user; +} + +function createNewRoom(client, roomId) { + var room = new Room(roomId, { + pendingEventOrdering: client._config.pendingEventOrdering + }); + reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); + + // we need to also re-emit room state and room member events, so hook it up + // to the client now. We need to add a listener for RoomState.members in + // order to hook them correctly. (TODO: find a better way?) + reEmit(client, room.currentState, [ + "RoomState.events", "RoomState.members", "RoomState.newMember" + ]); + room.currentState.on("RoomState.newMember", function(event, state, member) { + member.user = client.getUser(member.userId); + reEmit( + client, member, + [ + "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", + "RoomMember.membership" + ] + ); + }); + return room; +} + +function reEmit(reEmitEntity, emittableEntity, eventNames) { + utils.forEach(eventNames, function(eventName) { + // setup a listener on the entity (the Room, User, etc) for this event + emittableEntity.on(eventName, function() { + // take the args from the listener and reuse them, adding the + // event name to the arg list so it works with .emit() + // Transformation Example: + // listener on "foo" => function(a,b) { ... } + // Re-emit on "thing" => thing.emit("foo", a, b) + var newArgs = [eventName]; + for (var i = 0; i < arguments.length; i++) { + newArgs.push(arguments[i]); + } + reEmitEntity.emit.apply(reEmitEntity, newArgs); + }); + }); +} + +/** + * Internal class - unstable. + * Construct an entity which is able to sync with a homeserver. + * @param {MatrixClient} client The matrix client instance to use. + */ +function SyncApi(client) { + this.client = client; + this.opts = {}; +} + +/** + * @param {Object} opts + * @param {Number} opts.historyLen + * @param {Boolean} opts.includeArchived + */ +SyncApi.prototype.sync = function(opts) { + console.log("SyncApi.sync -> %s", opts); + this.opts = opts || {}; + return this._sync(); +} + + +SyncApi.prototype._prepareForSync = function(attempt) { + var client = this.client; + var self = this; + if (client.isGuest()) { + // no push rules for guests + this._sync(); + return; + } + + attempt = attempt || 1; + // we do push rules before syncing so when we gets events down we know immediately + // whether they are bing-worthy. + client.pushRules().done(function(result) { + client.pushRules = result; + self._sync(); + }, function(err) { + attempt += 1; + startSyncingRetryTimer(client, attempt, function() { + prepareForSync(client, attempt); + }); + updateSyncState(client, "ERROR", { error: err }); + }); +} + +SyncApi.prototype._sync = function(attempt) { + var opts = this.opts; + var client = this.client; + var self = this; + var historyLen = opts.historyLen; + var includeArchived = opts.includeArchived; + attempt = attempt || 1; + + var qps = { limit: historyLen }; + if (includeArchived) { + qps.archived = true; + } + if (client._guestRooms && client._isGuest) { + qps.room_id = JSON.stringify(client._guestRooms); + } + client._http.authedRequest( + undefined, "GET", "/initialSync", qps + ).done(function(data) { + var i, j; + // intercept the results and put them into our store + if (!(client.store instanceof StubStore)) { + utils.forEach( + utils.map(data.presence, EventMapper(client)), + function(e) { + var user = createNewUser(client, e.getContent().user_id); + user.setPresenceEvent(e); + client.store.storeUser(user); + }); + + // group receipts by room ID. + var receiptsByRoom = {}; + data.receipts = data.receipts || []; + utils.forEach(data.receipts.map(EventMapper(client)), + function(receiptEvent) { + if (!receiptsByRoom[receiptEvent.getRoomId()]) { + receiptsByRoom[receiptEvent.getRoomId()] = []; + } + receiptsByRoom[receiptEvent.getRoomId()].push(receiptEvent); + } + ); + + for (i = 0; i < data.rooms.length; i++) { + var room = createNewRoom(client, data.rooms[i].room_id); + if (!data.rooms[i].state) { + data.rooms[i].state = []; + } + if (data.rooms[i].membership === "invite") { + var inviteEvent = data.rooms[i].invite; + if (!inviteEvent) { + // fallback for servers which don't serve the invite key yet + inviteEvent = { + event_id: "$fake_" + room.roomId, + content: { + membership: "invite" + }, + state_key: client.credentials.userId, + user_id: data.rooms[i].inviter, + room_id: room.roomId, + type: "m.room.member" + }; + } + data.rooms[i].state.push(inviteEvent); + } + + _processRoomEvents( // XXX + client, room, data.rooms[i].state, data.rooms[i].messages + ); + + var receipts = receiptsByRoom[room.roomId] || []; + for (j = 0; j < receipts.length; j++) { + room.addReceipt(receipts[j]); + } + + var privateUserData = data.rooms[i].account_data || []; + var privateUserDataEvents = + utils.map(privateUserData, EventMapper(client)); + for (j = 0; j < privateUserDataEvents.length; j++) { + var event = privateUserDataEvents[j]; + if (event.getType() === "m.tag") { + room.addTags(event); + } + // XXX: unhandled private user data event - we should probably + // put it somewhere useful once the API has settled + } + + // cache the name/summary/etc prior to storage since we don't + // know how the store will serialise the Room. + room.recalculate(client.credentials.userId); + + client.store.storeRoom(room); + client.emit("Room", room); + } + } + + if (data) { + client.store.setSyncToken(data.end); + var events = []; + for (i = 0; i < data.presence.length; i++) { + events.push(new MatrixEvent(data.presence[i])); + } + for (i = 0; i < data.rooms.length; i++) { + if (data.rooms[i].state) { + for (j = 0; j < data.rooms[i].state.length; j++) { + events.push(new MatrixEvent(data.rooms[i].state[j])); + } + } + if (data.rooms[i].messages) { + for (j = 0; j < data.rooms[i].messages.chunk.length; j++) { + events.push( + new MatrixEvent(data.rooms[i].messages.chunk[j]) + ); + } + } + } + utils.forEach(events, function(e) { + client.emit("event", e); + }); + } + + client.clientRunning = true; + updateSyncState(client, "PREPARED"); + // assume success until we fail which may be 30+ secs + updateSyncState(client, "SYNCING"); + self._pollForEvents(); + }, function(err) { + console.error("/initialSync error (%s attempts): %s", attempt, err); + attempt += 1; + startSyncingRetryTimer(client, attempt, function() { + self._sync(opts, attempt); + }); + updateSyncState(client, "ERROR", { error: err }); + }); +}; + +/** + * This is an internal method. + * @param {MatrixClient} client + * @param {Number} attempt The attempt number + */ +SyncApi.prototype._pollForEvents = function(attempt) { + var client = this.client; + var self = this; + + attempt = attempt || 1; + + if (!client.clientRunning) { + return; + } + var timeoutMs = client._config.pollTimeout; + if (attempt > 1) { + // we think the connection is dead. If it comes back up, we won't know + // about it till /events returns. If the timeout= is high, this could + // be a long time. Set it to 1 when doing retries. + timeoutMs = 1; + } + var discardResult = false; + var timeoutObj = setTimeout(function() { + discardResult = true; + console.error("/events request timed out."); + self._pollForEvents(); + }, timeoutMs + (20 * 1000)); // 20s buffer + + var queryParams = { + from: client.store.getSyncToken(), + timeout: timeoutMs + }; + if (client._guestRooms && client._isGuest) { + queryParams.room_id = client._guestRooms; + } + + client._http.authedRequest(undefined, "GET", "/events", queryParams).done( + function(data) { + if (discardResult) { + return; + } + else { + clearTimeout(timeoutObj); + } + + if (client._syncState !== "SYNCING") { + updateSyncState(self, "SYNCING"); + } + + try { + var events = []; + if (data) { + events = utils.map(data.chunk, EventMapper(self)); + } + if (!(client.store instanceof StubStore)) { + var roomIdsWithNewInvites = {}; + // bucket events based on room. + var i = 0; + var roomIdToEvents = {}; + for (i = 0; i < events.length; i++) { + var roomId = events[i].getRoomId(); + // possible to have no room ID e.g. for presence events. + if (roomId) { + if (!roomIdToEvents[roomId]) { + roomIdToEvents[roomId] = []; + } + roomIdToEvents[roomId].push(events[i]); + if (events[i].getType() === "m.room.member" && + events[i].getContent().membership === "invite") { + roomIdsWithNewInvites[roomId] = true; + } + } + else if (events[i].getType() === "m.presence") { + var usr = client.store.getUser(events[i].getContent().user_id); + if (usr) { + usr.setPresenceEvent(events[i]); + } + else { + usr = createNewUser(self, events[i].getContent().user_id); + usr.setPresenceEvent(events[i]); + client.store.storeUser(usr); + } + } + } + + // add events to room + var roomIds = utils.keys(roomIdToEvents); + utils.forEach(roomIds, function(roomId) { + var room = client.store.getRoom(roomId); + var isBrandNewRoom = false; + if (!room) { + room = createNewRoom(self, roomId); + isBrandNewRoom = true; + } + + var wasJoined = room.hasMembershipState( + client.credentials.userId, "join" + ); + + room.addEvents(roomIdToEvents[roomId], "replace"); + room.recalculate(client.credentials.userId); + + // store the Room for things like invite events so developers + // can update the UI + if (isBrandNewRoom) { + client.store.storeRoom(room); + client.emit("Room", room); + } + + var justJoined = room.hasMembershipState( + client.credentials.userId, "join" + ); + + if (!wasJoined && justJoined) { + // we've just transitioned into a join state for this room, + // so sync state. + _syncRoom(self, room); // XXX + } + }); + + Object.keys(roomIdsWithNewInvites).forEach(function(inviteRoomId) { + _resolveInvites(self, client.store.getRoom(inviteRoomId)); // XXX + }); + } + if (data) { + client.store.setSyncToken(data.end); + utils.forEach(events, function(e) { + client.emit("event", e); + }); + } + } + catch (e) { + console.error("Event stream error:"); + console.error(e); + } + client._pollForEvents(); + }, function(err) { + console.error("/events error: %s", JSON.stringify(err)); + if (discardResult) { + return; + } + else { + clearTimeout(timeoutObj); + } + + attempt += 1; + startSyncingRetryTimer(self, attempt, function() { + client._pollForEvents(attempt); + }); + updateSyncState(self, "ERROR", { error: err }); + }); +} + +module.exports = SyncApi; From 3543abf7bd4077c3816bbbf93c9e1f5c731f6a01 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 10:42:05 +0000 Subject: [PATCH 02/32] Make things work again --- lib/client.js | 17 +++-- lib/pushprocessor.js | 2 +- lib/sync.js | 146 +++++++++++++++++++++++++++++++++++++------ 3 files changed, 134 insertions(+), 31 deletions(-) diff --git a/lib/client.js b/lib/client.js index e9c77469a..65665e308 100644 --- a/lib/client.js +++ b/lib/client.js @@ -621,9 +621,10 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) { this._http.authedRequest(undefined, "POST", path, undefined, {}).then( function(res) { var roomId = res.room_id; - var room = createNewRoom(self, roomId); // XXX FIXME TODO + var syncApi = new SyncApi(self); + var room = syncApi.createRoom(roomId); if (opts.syncRoom) { - return _syncRoom(self, room); // XXX FIXME TODO + return syncApi.syncRoom(room); } return q(room); }, function(err) { @@ -2424,12 +2425,6 @@ function checkTurnServers(client) { }); } -function retryTimeMsForAttempt(attempt) { - // 2,4,8,16,32,64,128,128,128,... seconds - // max 2^7 secs = 2.1 mins - return Math.pow(2, Math.min(attempt, 7)) * 1000; -} - function _reject(callback, defer, err) { if (callback) { callback(err); @@ -2456,6 +2451,10 @@ function _PojoToMatrixEventMapper(client) { return mapper; } +MatrixClient.prototype.getEventMapper = function() { + return _PojoToMatrixEventMapper(this); +}; + // Identity Server Operations // ========================== @@ -2502,8 +2501,6 @@ MatrixClient.prototype.generateClientSecret = function() { module.exports.MatrixClient = MatrixClient; /** */ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED; -/** */ -module.exports.EventMapper = _PojoToMatrixEventMapper; // MatrixClient Event JSDocs diff --git a/lib/pushprocessor.js b/lib/pushprocessor.js index 8ea138afa..e7c71029b 100644 --- a/lib/pushprocessor.js +++ b/lib/pushprocessor.js @@ -195,7 +195,7 @@ function PushProcessor(client) { }; var matchingRuleForEventWithRulesets = function(ev, rulesets) { - if (!rulesets) { return null; } + if (!rulesets || !rulesets.device) { return null; } if (ev.user_id == client.credentials.userId) { return null; } var allDevNames = Object.keys(rulesets.device); diff --git a/lib/sync.js b/lib/sync.js index c34e7a242..bd0c214d8 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -8,14 +8,19 @@ * an alternative syncing API, we may want to have a proper syncing interface * for HTTP and WS at some point. */ - +var q = require("q"); var StubStore = require("./store/stub"); -var EventMapper = require("./client").EventMapper; var User = require("./models/user"); var Room = require("./models/room"); var utils = require("./utils"); var MatrixEvent = require("./models/event").MatrixEvent; +function retryTimeMsForAttempt(attempt) { + // 2,4,8,16,32,64,128,128,128,... seconds + // max 2^7 secs = 2.1 mins + return Math.pow(2, Math.min(attempt, 7)) * 1000; +} + function startSyncingRetryTimer(client, attempt, fn) { client._syncingRetry = {}; client._syncingRetry.fn = fn; @@ -79,6 +84,99 @@ function reEmit(reEmitEntity, emittableEntity, eventNames) { }); } +function _syncRoom(client, room) { + if (client._syncingRooms[room.roomId]) { + return client._syncingRooms[room.roomId]; + } + var defer = q.defer(); + client._syncingRooms[room.roomId] = defer.promise; + client.roomInitialSync(room.roomId, client._config.initialSyncLimit).done( + function(res) { + room.timeline = []; // blow away any previous messages. + _processRoomEvents(client, room, res.state, res.messages); + room.recalculate(client.credentials.userId); + client.store.storeRoom(room); + client.emit("Room", room); + defer.resolve(room); + client._syncingRooms[room.roomId] = undefined; + }, function(err) { + defer.reject(err); + client._syncingRooms[room.roomId] = undefined; + }); + return defer.promise; +} + +function _processRoomEvents(client, room, stateEventList, messageChunk) { + // "old" and "current" state are the same initially; they + // start diverging if the user paginates. + // We must deep copy otherwise membership changes in old state + // will leak through to current state! + var oldStateEvents = utils.map( + utils.deepCopy(stateEventList), client.getEventMapper() + ); + var stateEvents = utils.map(stateEventList, client.getEventMapper()); + room.oldState.setStateEvents(oldStateEvents); + room.currentState.setStateEvents(stateEvents); + + _resolveInvites(client, room); + + // add events to the timeline *after* setting the state + // events so messages use the right display names. Initial sync + // returns messages in chronological order, so we need to reverse + // it to get most recent -> oldest. We need it in that order in + // order to diverge old/current state correctly. + room.addEventsToTimeline( + utils.map( + messageChunk ? messageChunk.chunk : [], + client.getEventMapper() + ).reverse(), true + ); + if (messageChunk) { + room.oldState.paginationToken = messageChunk.start; + } +} + +function _resolveInvites(client, room) { + if (!room || !client._config.resolveInvitesToProfiles) { + return; + } + // For each invited room member we want to give them a displayname/avatar url + // if they have one (the m.room.member invites don't contain this). + room.getMembersWithMembership("invite").forEach(function(member) { + if (member._requestedProfileInfo) { + return; + } + member._requestedProfileInfo = true; + // try to get a cached copy first. + var user = client.getUser(member.userId); + var promise; + if (user) { + promise = q({ + avatar_url: user.avatarUrl, + displayname: user.displayName + }); + } + else { + promise = client.getProfileInfo(member.userId); + } + promise.done(function(info) { + // slightly naughty by doctoring the invite event but this means all + // the code paths remain the same between invite/join display name stuff + // which is a worthy trade-off for some minor pollution. + var inviteEvent = member.events.member; + if (inviteEvent.getContent().membership !== "invite") { + // between resolving and now they have since joined, so don't clobber + return; + } + inviteEvent.getContent().avatar_url = info.avatar_url; + inviteEvent.getContent().displayname = info.displayname; + member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners + }, function(err) { + // OH WELL. + }); + }); +} + /** * Internal class - unstable. * Construct an entity which is able to sync with a homeserver. @@ -89,15 +187,23 @@ function SyncApi(client) { this.opts = {}; } +SyncApi.prototype.createRoom = function(roomId) { + return createNewRoom(this.client, roomId); +}; + +SyncApi.prototype.syncRoom = function(room) { + return _syncRoom(this.client, room); +}; + /** * @param {Object} opts * @param {Number} opts.historyLen * @param {Boolean} opts.includeArchived */ SyncApi.prototype.sync = function(opts) { - console.log("SyncApi.sync -> %s", opts); + console.log("SyncApi.sync"); this.opts = opts || {}; - return this._sync(); + return this._prepareForSync(); } @@ -119,7 +225,7 @@ SyncApi.prototype._prepareForSync = function(attempt) { }, function(err) { attempt += 1; startSyncingRetryTimer(client, attempt, function() { - prepareForSync(client, attempt); + self._prepareForSync(attempt); }); updateSyncState(client, "ERROR", { error: err }); }); @@ -147,7 +253,7 @@ SyncApi.prototype._sync = function(attempt) { // intercept the results and put them into our store if (!(client.store instanceof StubStore)) { utils.forEach( - utils.map(data.presence, EventMapper(client)), + utils.map(data.presence, client.getEventMapper()), function(e) { var user = createNewUser(client, e.getContent().user_id); user.setPresenceEvent(e); @@ -157,7 +263,7 @@ SyncApi.prototype._sync = function(attempt) { // group receipts by room ID. var receiptsByRoom = {}; data.receipts = data.receipts || []; - utils.forEach(data.receipts.map(EventMapper(client)), + utils.forEach(data.receipts.map(client.getEventMapper()), function(receiptEvent) { if (!receiptsByRoom[receiptEvent.getRoomId()]) { receiptsByRoom[receiptEvent.getRoomId()] = []; @@ -189,7 +295,7 @@ SyncApi.prototype._sync = function(attempt) { data.rooms[i].state.push(inviteEvent); } - _processRoomEvents( // XXX + _processRoomEvents( client, room, data.rooms[i].state, data.rooms[i].messages ); @@ -200,7 +306,7 @@ SyncApi.prototype._sync = function(attempt) { var privateUserData = data.rooms[i].account_data || []; var privateUserDataEvents = - utils.map(privateUserData, EventMapper(client)); + utils.map(privateUserData, client.getEventMapper()); for (j = 0; j < privateUserDataEvents.length; j++) { var event = privateUserDataEvents[j]; if (event.getType() === "m.tag") { @@ -253,7 +359,7 @@ SyncApi.prototype._sync = function(attempt) { console.error("/initialSync error (%s attempts): %s", attempt, err); attempt += 1; startSyncingRetryTimer(client, attempt, function() { - self._sync(opts, attempt); + self._sync(attempt); }); updateSyncState(client, "ERROR", { error: err }); }); @@ -305,13 +411,13 @@ SyncApi.prototype._pollForEvents = function(attempt) { } if (client._syncState !== "SYNCING") { - updateSyncState(self, "SYNCING"); + updateSyncState(client, "SYNCING"); } try { var events = []; if (data) { - events = utils.map(data.chunk, EventMapper(self)); + events = utils.map(data.chunk, client.getEventMapper()); } if (!(client.store instanceof StubStore)) { var roomIdsWithNewInvites = {}; @@ -337,7 +443,7 @@ SyncApi.prototype._pollForEvents = function(attempt) { usr.setPresenceEvent(events[i]); } else { - usr = createNewUser(self, events[i].getContent().user_id); + usr = createNewUser(client, events[i].getContent().user_id); usr.setPresenceEvent(events[i]); client.store.storeUser(usr); } @@ -350,7 +456,7 @@ SyncApi.prototype._pollForEvents = function(attempt) { var room = client.store.getRoom(roomId); var isBrandNewRoom = false; if (!room) { - room = createNewRoom(self, roomId); + room = createNewRoom(client, roomId); isBrandNewRoom = true; } @@ -375,12 +481,12 @@ SyncApi.prototype._pollForEvents = function(attempt) { if (!wasJoined && justJoined) { // we've just transitioned into a join state for this room, // so sync state. - _syncRoom(self, room); // XXX + _syncRoom(client, room); } }); Object.keys(roomIdsWithNewInvites).forEach(function(inviteRoomId) { - _resolveInvites(self, client.store.getRoom(inviteRoomId)); // XXX + _resolveInvites(client, client.store.getRoom(inviteRoomId)); }); } if (data) { @@ -394,7 +500,7 @@ SyncApi.prototype._pollForEvents = function(attempt) { console.error("Event stream error:"); console.error(e); } - client._pollForEvents(); + self._pollForEvents(); }, function(err) { console.error("/events error: %s", JSON.stringify(err)); if (discardResult) { @@ -405,10 +511,10 @@ SyncApi.prototype._pollForEvents = function(attempt) { } attempt += 1; - startSyncingRetryTimer(self, attempt, function() { - client._pollForEvents(attempt); + startSyncingRetryTimer(client, attempt, function() { + self._pollForEvents(attempt); }); - updateSyncState(self, "ERROR", { error: err }); + updateSyncState(client, "ERROR", { error: err }); }); } From 529fb2355535a0233f2057552844666f6888600e Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 10:43:21 +0000 Subject: [PATCH 03/32] Linting --- lib/sync.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index bd0c214d8..b3e266aee 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -204,7 +204,7 @@ SyncApi.prototype.sync = function(opts) { console.log("SyncApi.sync"); this.opts = opts || {}; return this._prepareForSync(); -} +}; SyncApi.prototype._prepareForSync = function(attempt) { @@ -229,7 +229,7 @@ SyncApi.prototype._prepareForSync = function(attempt) { }); updateSyncState(client, "ERROR", { error: err }); }); -} +}; SyncApi.prototype._sync = function(attempt) { var opts = this.opts; @@ -516,6 +516,6 @@ SyncApi.prototype._pollForEvents = function(attempt) { }); updateSyncState(client, "ERROR", { error: err }); }); -} +}; module.exports = SyncApi; From 0576e4ca0cfb27395731260b4b69125bb6d9dd88 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 11:16:46 +0000 Subject: [PATCH 04/32] Linting --- lib/client.js | 3 +++ lib/sync.js | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/client.js b/lib/client.js index 65665e308..ecd0e4d41 100644 --- a/lib/client.js +++ b/lib/client.js @@ -2451,6 +2451,9 @@ function _PojoToMatrixEventMapper(client) { return mapper; } +/** + * @return {Function} + */ MatrixClient.prototype.getEventMapper = function() { return _PojoToMatrixEventMapper(this); }; diff --git a/lib/sync.js b/lib/sync.js index b3e266aee..efae7149d 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -187,10 +187,18 @@ function SyncApi(client) { this.opts = {}; } +/** + * @param {string} roomId + * @return {Room} + */ SyncApi.prototype.createRoom = function(roomId) { return createNewRoom(this.client, roomId); }; +/** + * @param {Room} room + * @return {Promise} + */ SyncApi.prototype.syncRoom = function(room) { return _syncRoom(this.client, room); }; @@ -203,10 +211,13 @@ SyncApi.prototype.syncRoom = function(room) { SyncApi.prototype.sync = function(opts) { console.log("SyncApi.sync"); this.opts = opts || {}; - return this._prepareForSync(); + this._prepareForSync(); }; +/** + * @param {Number=} attempt + */ SyncApi.prototype._prepareForSync = function(attempt) { var client = this.client; var self = this; @@ -231,6 +242,9 @@ SyncApi.prototype._prepareForSync = function(attempt) { }); }; +/** + * @param {Number=} attempt + */ SyncApi.prototype._sync = function(attempt) { var opts = this.opts; var client = this.client; @@ -367,15 +381,14 @@ SyncApi.prototype._sync = function(attempt) { /** * This is an internal method. - * @param {MatrixClient} client - * @param {Number} attempt The attempt number + * @param {Number=} attempt The attempt number */ SyncApi.prototype._pollForEvents = function(attempt) { var client = this.client; var self = this; attempt = attempt || 1; - + if (!client.clientRunning) { return; } @@ -518,4 +531,5 @@ SyncApi.prototype._pollForEvents = function(attempt) { }); }; +/** */ module.exports = SyncApi; From f7ffed4b98f572afe182fc937a48e7d766c130a4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 11:19:20 +0000 Subject: [PATCH 05/32] More linting --- lib/sync.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sync.js b/lib/sync.js index efae7149d..2735beedd 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -180,6 +180,7 @@ function _resolveInvites(client, room) { /** * Internal class - unstable. * Construct an entity which is able to sync with a homeserver. + * @constructor * @param {MatrixClient} client The matrix client instance to use. */ function SyncApi(client) { From 06f927aa2294a853fe4b3af60f21759ee11f2a30 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 11:27:03 +0000 Subject: [PATCH 06/32] Minor cleanup and reshuffle --- lib/client.js | 18 +-- lib/sync.js | 327 +++++++++++++++++++++++++------------------------- 2 files changed, 170 insertions(+), 175 deletions(-) diff --git a/lib/client.js b/lib/client.js index ecd0e4d41..f655bdacb 100644 --- a/lib/client.js +++ b/lib/client.js @@ -2171,18 +2171,6 @@ MatrixClient.prototype.isLoggedIn = function() { // reconnect after connectivity outages -/** - * This is an internal method. - * @param {MatrixClient} client - */ -function doSync(client) { - var syncApi = new SyncApi(client); - syncApi.sync({ - historyLen: client._config.initialSyncLimit, - includeArchived: client._config.includeArchivedRooms - }); -} - /** * High level helper method to call initialSync, emit the resulting events, * and then start polling the eventStream for new events. To listen for these @@ -2231,7 +2219,11 @@ MatrixClient.prototype.startClient = function(opts) { // periodically poll for turn servers if we support voip checkTurnServers(this); - doSync(this); + var syncApi = new SyncApi(this); + syncApi.sync({ + historyLen: this._config.initialSyncLimit, + includeArchived: this._config.includeArchivedRooms + }); }; /** diff --git a/lib/sync.js b/lib/sync.js index 2735beedd..b26af98da 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -15,168 +15,6 @@ var Room = require("./models/room"); var utils = require("./utils"); var MatrixEvent = require("./models/event").MatrixEvent; -function retryTimeMsForAttempt(attempt) { - // 2,4,8,16,32,64,128,128,128,... seconds - // max 2^7 secs = 2.1 mins - return Math.pow(2, Math.min(attempt, 7)) * 1000; -} - -function startSyncingRetryTimer(client, attempt, fn) { - client._syncingRetry = {}; - client._syncingRetry.fn = fn; - client._syncingRetry.timeoutId = setTimeout(function() { - fn(); - }, retryTimeMsForAttempt(attempt)); -} - -function updateSyncState(client, newState, data) { - var old = client._syncState; - client._syncState = newState; - client.emit("sync", client._syncState, old, data); -} - -function createNewUser(client, userId) { - var user = new User(userId); - reEmit(client, user, ["User.avatarUrl", "User.displayName", "User.presence"]); - return user; -} - -function createNewRoom(client, roomId) { - var room = new Room(roomId, { - pendingEventOrdering: client._config.pendingEventOrdering - }); - reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); - - // we need to also re-emit room state and room member events, so hook it up - // to the client now. We need to add a listener for RoomState.members in - // order to hook them correctly. (TODO: find a better way?) - reEmit(client, room.currentState, [ - "RoomState.events", "RoomState.members", "RoomState.newMember" - ]); - room.currentState.on("RoomState.newMember", function(event, state, member) { - member.user = client.getUser(member.userId); - reEmit( - client, member, - [ - "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", - "RoomMember.membership" - ] - ); - }); - return room; -} - -function reEmit(reEmitEntity, emittableEntity, eventNames) { - utils.forEach(eventNames, function(eventName) { - // setup a listener on the entity (the Room, User, etc) for this event - emittableEntity.on(eventName, function() { - // take the args from the listener and reuse them, adding the - // event name to the arg list so it works with .emit() - // Transformation Example: - // listener on "foo" => function(a,b) { ... } - // Re-emit on "thing" => thing.emit("foo", a, b) - var newArgs = [eventName]; - for (var i = 0; i < arguments.length; i++) { - newArgs.push(arguments[i]); - } - reEmitEntity.emit.apply(reEmitEntity, newArgs); - }); - }); -} - -function _syncRoom(client, room) { - if (client._syncingRooms[room.roomId]) { - return client._syncingRooms[room.roomId]; - } - var defer = q.defer(); - client._syncingRooms[room.roomId] = defer.promise; - client.roomInitialSync(room.roomId, client._config.initialSyncLimit).done( - function(res) { - room.timeline = []; // blow away any previous messages. - _processRoomEvents(client, room, res.state, res.messages); - room.recalculate(client.credentials.userId); - client.store.storeRoom(room); - client.emit("Room", room); - defer.resolve(room); - client._syncingRooms[room.roomId] = undefined; - }, function(err) { - defer.reject(err); - client._syncingRooms[room.roomId] = undefined; - }); - return defer.promise; -} - -function _processRoomEvents(client, room, stateEventList, messageChunk) { - // "old" and "current" state are the same initially; they - // start diverging if the user paginates. - // We must deep copy otherwise membership changes in old state - // will leak through to current state! - var oldStateEvents = utils.map( - utils.deepCopy(stateEventList), client.getEventMapper() - ); - var stateEvents = utils.map(stateEventList, client.getEventMapper()); - room.oldState.setStateEvents(oldStateEvents); - room.currentState.setStateEvents(stateEvents); - - _resolveInvites(client, room); - - // add events to the timeline *after* setting the state - // events so messages use the right display names. Initial sync - // returns messages in chronological order, so we need to reverse - // it to get most recent -> oldest. We need it in that order in - // order to diverge old/current state correctly. - room.addEventsToTimeline( - utils.map( - messageChunk ? messageChunk.chunk : [], - client.getEventMapper() - ).reverse(), true - ); - if (messageChunk) { - room.oldState.paginationToken = messageChunk.start; - } -} - -function _resolveInvites(client, room) { - if (!room || !client._config.resolveInvitesToProfiles) { - return; - } - // For each invited room member we want to give them a displayname/avatar url - // if they have one (the m.room.member invites don't contain this). - room.getMembersWithMembership("invite").forEach(function(member) { - if (member._requestedProfileInfo) { - return; - } - member._requestedProfileInfo = true; - // try to get a cached copy first. - var user = client.getUser(member.userId); - var promise; - if (user) { - promise = q({ - avatar_url: user.avatarUrl, - displayname: user.displayName - }); - } - else { - promise = client.getProfileInfo(member.userId); - } - promise.done(function(info) { - // slightly naughty by doctoring the invite event but this means all - // the code paths remain the same between invite/join display name stuff - // which is a worthy trade-off for some minor pollution. - var inviteEvent = member.events.member; - if (inviteEvent.getContent().membership !== "invite") { - // between resolving and now they have since joined, so don't clobber - return; - } - inviteEvent.getContent().avatar_url = info.avatar_url; - inviteEvent.getContent().displayname = info.displayname; - member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners - }, function(err) { - // OH WELL. - }); - }); -} - /** * Internal class - unstable. * Construct an entity which is able to sync with a homeserver. @@ -205,6 +43,7 @@ SyncApi.prototype.syncRoom = function(room) { }; /** + * Main entry point * @param {Object} opts * @param {Number} opts.historyLen * @param {Boolean} opts.includeArchived @@ -532,5 +371,169 @@ SyncApi.prototype._pollForEvents = function(attempt) { }); }; + + +function retryTimeMsForAttempt(attempt) { + // 2,4,8,16,32,64,128,128,128,... seconds + // max 2^7 secs = 2.1 mins + return Math.pow(2, Math.min(attempt, 7)) * 1000; +} + +function startSyncingRetryTimer(client, attempt, fn) { + client._syncingRetry = {}; + client._syncingRetry.fn = fn; + client._syncingRetry.timeoutId = setTimeout(function() { + fn(); + }, retryTimeMsForAttempt(attempt)); +} + +function updateSyncState(client, newState, data) { + var old = client._syncState; + client._syncState = newState; + client.emit("sync", client._syncState, old, data); +} + +function createNewUser(client, userId) { + var user = new User(userId); + reEmit(client, user, ["User.avatarUrl", "User.displayName", "User.presence"]); + return user; +} + +function createNewRoom(client, roomId) { + var room = new Room(roomId, { + pendingEventOrdering: client._config.pendingEventOrdering + }); + reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); + + // we need to also re-emit room state and room member events, so hook it up + // to the client now. We need to add a listener for RoomState.members in + // order to hook them correctly. (TODO: find a better way?) + reEmit(client, room.currentState, [ + "RoomState.events", "RoomState.members", "RoomState.newMember" + ]); + room.currentState.on("RoomState.newMember", function(event, state, member) { + member.user = client.getUser(member.userId); + reEmit( + client, member, + [ + "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", + "RoomMember.membership" + ] + ); + }); + return room; +} + +function reEmit(reEmitEntity, emittableEntity, eventNames) { + utils.forEach(eventNames, function(eventName) { + // setup a listener on the entity (the Room, User, etc) for this event + emittableEntity.on(eventName, function() { + // take the args from the listener and reuse them, adding the + // event name to the arg list so it works with .emit() + // Transformation Example: + // listener on "foo" => function(a,b) { ... } + // Re-emit on "thing" => thing.emit("foo", a, b) + var newArgs = [eventName]; + for (var i = 0; i < arguments.length; i++) { + newArgs.push(arguments[i]); + } + reEmitEntity.emit.apply(reEmitEntity, newArgs); + }); + }); +} + +function _syncRoom(client, room) { + if (client._syncingRooms[room.roomId]) { + return client._syncingRooms[room.roomId]; + } + var defer = q.defer(); + client._syncingRooms[room.roomId] = defer.promise; + client.roomInitialSync(room.roomId, client._config.initialSyncLimit).done( + function(res) { + room.timeline = []; // blow away any previous messages. + _processRoomEvents(client, room, res.state, res.messages); + room.recalculate(client.credentials.userId); + client.store.storeRoom(room); + client.emit("Room", room); + defer.resolve(room); + client._syncingRooms[room.roomId] = undefined; + }, function(err) { + defer.reject(err); + client._syncingRooms[room.roomId] = undefined; + }); + return defer.promise; +} + +function _processRoomEvents(client, room, stateEventList, messageChunk) { + // "old" and "current" state are the same initially; they + // start diverging if the user paginates. + // We must deep copy otherwise membership changes in old state + // will leak through to current state! + var oldStateEvents = utils.map( + utils.deepCopy(stateEventList), client.getEventMapper() + ); + var stateEvents = utils.map(stateEventList, client.getEventMapper()); + room.oldState.setStateEvents(oldStateEvents); + room.currentState.setStateEvents(stateEvents); + + _resolveInvites(client, room); + + // add events to the timeline *after* setting the state + // events so messages use the right display names. Initial sync + // returns messages in chronological order, so we need to reverse + // it to get most recent -> oldest. We need it in that order in + // order to diverge old/current state correctly. + room.addEventsToTimeline( + utils.map( + messageChunk ? messageChunk.chunk : [], + client.getEventMapper() + ).reverse(), true + ); + if (messageChunk) { + room.oldState.paginationToken = messageChunk.start; + } +} + +function _resolveInvites(client, room) { + if (!room || !client._config.resolveInvitesToProfiles) { + return; + } + // For each invited room member we want to give them a displayname/avatar url + // if they have one (the m.room.member invites don't contain this). + room.getMembersWithMembership("invite").forEach(function(member) { + if (member._requestedProfileInfo) { + return; + } + member._requestedProfileInfo = true; + // try to get a cached copy first. + var user = client.getUser(member.userId); + var promise; + if (user) { + promise = q({ + avatar_url: user.avatarUrl, + displayname: user.displayName + }); + } + else { + promise = client.getProfileInfo(member.userId); + } + promise.done(function(info) { + // slightly naughty by doctoring the invite event but this means all + // the code paths remain the same between invite/join display name stuff + // which is a worthy trade-off for some minor pollution. + var inviteEvent = member.events.member; + if (inviteEvent.getContent().membership !== "invite") { + // between resolving and now they have since joined, so don't clobber + return; + } + inviteEvent.getContent().avatar_url = info.avatar_url; + inviteEvent.getContent().displayname = info.displayname; + member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners + }, function(err) { + // OH WELL. + }); + }); +} + /** */ module.exports = SyncApi; From b622960b32d97df55f8dde068b54022f2bdf6a14 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 15:25:09 +0000 Subject: [PATCH 07/32] Do all prep for /sync calls This includes managing filters in localStorage. The /sync response is not yet parsed. --- lib/client.js | 21 +----- lib/matrix.js | 4 +- lib/store/memory.js | 39 +++++++++- lib/store/stub.js | 9 +++ lib/sync.js | 174 ++++++++++++++++++++++++++++++++------------ 5 files changed, 180 insertions(+), 67 deletions(-) diff --git a/lib/client.js b/lib/client.js index f655bdacb..d54140b67 100644 --- a/lib/client.js +++ b/lib/client.js @@ -31,9 +31,6 @@ try { // Olm not installed. } -// TODO: -// Internal: rate limiting - var OLM_ALGORITHM = "m.olm.v1.curve25519-aes-sha2"; /** @@ -147,7 +144,6 @@ function MatrixClient(opts) { this.callList = { // callId: MatrixCall }; - this._config = {}; // see startClient() // try constructing a MatrixCall to see if we are running in an environment // which has WebRTC. If we are, listen for and handle m.call.* events. @@ -2197,6 +2193,7 @@ MatrixClient.prototype.startClient = function(opts) { // client is already running. return; } + this.clientRunning = true; // backwards compat for when 'opts' was 'historyLen'. if (typeof opts === "number") { opts = { @@ -2204,14 +2201,6 @@ MatrixClient.prototype.startClient = function(opts) { }; } - opts = opts || {}; - opts.initialSyncLimit = opts.initialSyncLimit || 8; - opts.includeArchivedRooms = opts.includeArchivedRooms || false; - opts.resolveInvitesToProfiles = opts.resolveInvitesToProfiles || false; - opts.pollTimeout = opts.pollTimeout || (30 * 1000); - opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological"; - this._config = opts; - if (CRYPTO_ENABLED && this.sessionStore !== null) { this.uploadKeys(5); } @@ -2219,11 +2208,8 @@ MatrixClient.prototype.startClient = function(opts) { // periodically poll for turn servers if we support voip checkTurnServers(this); - var syncApi = new SyncApi(this); - syncApi.sync({ - historyLen: this._config.initialSyncLimit, - includeArchived: this._config.includeArchivedRooms - }); + var syncApi = new SyncApi(this, opts); + syncApi.sync(); }; /** @@ -2233,6 +2219,7 @@ MatrixClient.prototype.startClient = function(opts) { MatrixClient.prototype.stopClient = function() { this.clientRunning = false; // TODO: f.e. Room => self.store.storeRoom(room) ? + // TODO: Actually stop the SyncApi }; function setupCallEventHandler(client) { diff --git a/lib/matrix.js b/lib/matrix.js index f150f7e76..a68350488 100644 --- a/lib/matrix.js +++ b/lib/matrix.js @@ -81,7 +81,9 @@ module.exports.createClient = function(opts) { }; } opts.request = opts.request || request; - opts.store = opts.store || new module.exports.MatrixInMemoryStore(); + opts.store = opts.store || new module.exports.MatrixInMemoryStore({ + localStorage: global.localStorage + }); opts.scheduler = opts.scheduler || new module.exports.MatrixScheduler(); return new module.exports.MatrixClient(opts); }; diff --git a/lib/store/memory.js b/lib/store/memory.js index ae8257f03..79cc25cae 100644 --- a/lib/store/memory.js +++ b/lib/store/memory.js @@ -8,8 +8,13 @@ /** * Construct a new in-memory data store for the Matrix Client. * @constructor + * @param {Object=} opts Config options + * @param {LocalStorage} opts.localStorage The local storage instance to persist some forms + * of data such as tokens. Rooms will NOT be stored. See {@link WebStorageStore} to + * persist rooms. */ -module.exports.MatrixInMemoryStore = function MatrixInMemoryStore() { +module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) { + opts = opts || {}; this.rooms = { // roomId: Room }; @@ -22,6 +27,7 @@ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore() { // filterId: Filter // } }; + this.localStorage = opts.localStorage; }; module.exports.MatrixInMemoryStore.prototype = { @@ -139,6 +145,37 @@ module.exports.MatrixInMemoryStore.prototype = { return null; } return this.filters[userId][filterId]; + }, + + /** + * Retrieve a filter ID with the given name. + * @param {string} filterName The filter name. + * @return {?string} The filter ID or null. + */ + getFilterIdByName: function(filterName) { + if (!this.localStorage) { + return null; + } + try { + return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName); + } + catch(e) {} + return null; + }, + + /** + * Set a filter name to ID mapping. + * @param {string} filterName + * @param {string} filterId + */ + setFilterIdByName: function(filterName, filterId) { + if (!this.localStorage) { + return; + } + try { + this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId); + } + catch (e) {} } // TODO diff --git a/lib/store/stub.js b/lib/store/stub.js index a8f0aa9bc..1c835eca9 100644 --- a/lib/store/stub.js +++ b/lib/store/stub.js @@ -113,6 +113,15 @@ StubStore.prototype = { */ getFilter: function(userId, filterId) { return null; + }, + + /** + * Retrieve a filter ID with the given name. + * @param {string} filterName The filter name. + * @return {?string} The filter ID or null. + */ + getFilterIdByName: function(filterName) { + return null; } // TODO diff --git a/lib/sync.js b/lib/sync.js index b26af98da..1e7cb00af 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -14,6 +14,10 @@ var User = require("./models/user"); var Room = require("./models/room"); var utils = require("./utils"); var MatrixEvent = require("./models/event").MatrixEvent; +var httpApi = require("./http-api"); +var Filter = require("./filter"); + +var FILTER_SYNC = "FILTER_SYNC"; /** * Internal class - unstable. @@ -21,9 +25,15 @@ var MatrixEvent = require("./models/event").MatrixEvent; * @constructor * @param {MatrixClient} client The matrix client instance to use. */ -function SyncApi(client) { +function SyncApi(client, opts) { this.client = client; - this.opts = {}; + opts = opts || {}; + opts.initialSyncLimit = opts.initialSyncLimit || 8; + opts.includeArchivedRooms = opts.includeArchivedRooms || false; + opts.resolveInvitesToProfiles = opts.resolveInvitesToProfiles || false; + opts.pollTimeout = opts.pollTimeout || (30 * 1000); + opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological"; + this.opts = opts; } /** @@ -44,65 +54,128 @@ SyncApi.prototype.syncRoom = function(room) { /** * Main entry point - * @param {Object} opts - * @param {Number} opts.historyLen - * @param {Boolean} opts.includeArchived */ -SyncApi.prototype.sync = function(opts) { +SyncApi.prototype.sync = function() { console.log("SyncApi.sync"); - this.opts = opts || {}; - this._prepareForSync(); -}; - - -/** - * @param {Number=} attempt - */ -SyncApi.prototype._prepareForSync = function(attempt) { var client = this.client; var self = this; + + // We need to do one-off checks before we can begin the /sync loop. + // These are: + // 1) We need to get push rules so we can check if events should bing as we get + // them from /sync. + // 2) We need to get/create a filter which we can use for /sync. + + function getPushRules(attempt) { + attempt = attempt || 0; + attempt += 1; + + client.pushRules().done(function(result) { + console.log("Got push rules"); + client.pushRules = result; + getFilter(); // Now get the filter + }, retryHandler(attempt, getPushRules)); + } + + function getFilter(attempt) { + attempt = attempt || 0; + attempt += 1; + + + // Get or create filter + var filterId = client.store.getFilterIdByName(FILTER_SYNC); + if (filterId) { + // super, just use that. + console.log("Using existing filter ID %s", filterId); + self._sync({ filterId: filterId }); + return; + } + + // create a filter + var filter = new Filter(client.credentials.userId); + filter.setTimelineLimit(self.opts.initialSyncLimit); + client.createFilter(filter.getDefinition()).done(function(filter) { + client.store.setFilterIdByName(FILTER_SYNC, filter.filterId); + console.log("Created filter ", filter.filterId); + self._sync({ filterId: filter.filterId }); // Now start the /sync loop + }, retryHandler(attempt, getFilter)); + } + + // sets the sync state to error and waits a bit before re-invoking the function. + function retryHandler(attempt, fnToRun) { + return function(err) { + startSyncingRetryTimer(client, attempt, function() { + fnToRun(attempt); + }); + updateSyncState(client, "ERROR", { error: err }); + }; + } + if (client.isGuest()) { // no push rules for guests - this._sync(); - return; + getFilter(); + } + else { + getPushRules(); } - - attempt = attempt || 1; - // we do push rules before syncing so when we gets events down we know immediately - // whether they are bing-worthy. - client.pushRules().done(function(result) { - client.pushRules = result; - self._sync(); - }, function(err) { - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - self._prepareForSync(attempt); - }); - updateSyncState(client, "ERROR", { error: err }); - }); }; /** + * Invoke me to do /sync calls + * @param {Object} syncOptions + * @param {string} syncOptions.filterId + * @param {boolean} syncOptions.hasSyncedBefore * @param {Number=} attempt */ -SyncApi.prototype._sync = function(attempt) { - var opts = this.opts; +SyncApi.prototype._sync = function(syncOptions, attempt) { var client = this.client; var self = this; - var historyLen = opts.historyLen; - var includeArchived = opts.includeArchived; attempt = attempt || 1; - var qps = { limit: historyLen }; - if (includeArchived) { - qps.archived = true; - } + // TODO include archived rooms flag. + + var qps = { + filter: syncOptions.filterId, + timeout: this.opts.pollTimeout, + since: client.store.getSyncToken() || undefined // do not send 'null' + }; + if (client._guestRooms && client._isGuest) { qps.room_id = JSON.stringify(client._guestRooms); } - client._http.authedRequest( - undefined, "GET", "/initialSync", qps + + client._http.authedRequestWithPrefix( + undefined, "GET", "/sync", qps, undefined, httpApi.PREFIX_V2_ALPHA ).done(function(data) { + // data looks like: + // { + // next_batch: $token, + // presence: [PresencEvents], + // rooms: { + // invite: { + // $roomid: { + // invite_state: { events: [] } + // } + // }, + // join: { + // $roomid: { + // state: { events: [] }, + // timeline: { events: [], prev_batch: $token, limited: true }, + // ephemeral: { events: [] }, + // account_data: { events: [] } + // } + // }, + // leave: { + // $roomid: { + // state: { events: [] }, + // timeline: { events: [], prev_batch: $token } + // } + // } + // } + // } + console.log("Got data %s", data); + + /* var i, j; // intercept the results and put them into our store if (!(client.store instanceof StubStore)) { @@ -180,7 +253,6 @@ SyncApi.prototype._sync = function(attempt) { } if (data) { - client.store.setSyncToken(data.end); var events = []; for (i = 0; i < data.presence.length; i++) { events.push(new MatrixEvent(data.presence[i])); @@ -204,16 +276,22 @@ SyncApi.prototype._sync = function(attempt) { }); } - client.clientRunning = true; - updateSyncState(client, "PREPARED"); - // assume success until we fail which may be 30+ secs + + // assume success until we fail which may be 30+ secs */ + + client.store.setSyncToken(data.next_batch); + + if (!syncOptions.hasSyncedBefore) { + updateSyncState(client, "PREPARED"); + syncOptions.hasSyncedBefore = true; + } updateSyncState(client, "SYNCING"); - self._pollForEvents(); + self._sync(syncOptions); }, function(err) { - console.error("/initialSync error (%s attempts): %s", attempt, err); + console.error("/sync error (%s attempts): %s", attempt, err); attempt += 1; startSyncingRetryTimer(client, attempt, function() { - self._sync(attempt); + self._sync(syncOptions, attempt); }); updateSyncState(client, "ERROR", { error: err }); }); From 243bdd78f42cc6f10ef7291dba9d34bcad184ef6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 16:09:47 +0000 Subject: [PATCH 08/32] Handle presence key in /sync --- lib/models/event.js | 2 +- lib/sync.js | 18 ++++++++++++- lib/utils.js | 63 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/lib/models/event.js b/lib/models/event.js index 18c2fe068..dd0d4623e 100644 --- a/lib/models/event.js +++ b/lib/models/event.js @@ -63,7 +63,7 @@ module.exports.MatrixEvent.prototype = { * @return {string} The user ID, e.g. @alice:matrix.org */ getSender: function() { - return this.event.user_id; + return this.event.sender || this.event.user_id; // v2 / v1 }, /** diff --git a/lib/sync.js b/lib/sync.js index 1e7cb00af..26c604b4e 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -150,7 +150,7 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // data looks like: // { // next_batch: $token, - // presence: [PresencEvents], + // presence: { events: [] }, // rooms: { // invite: { // $roomid: { @@ -175,6 +175,22 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // } console.log("Got data %s", data); + // handle presence events (User objects) + if (data.presence && utils.isArray(data.presence.events)) { + data.presence.events.map(client.getEventMapper()).forEach(function(presenceEvent) { + var user = client.store.getUser(presenceEvent.getSender()); + if (user) { + user.setPresenceEvent(presenceEvent); + } + else { + user = createNewUser(client, presenceEvent.getSender()); + user.setPresenceEvent(presenceEvent); + client.store.storeUser(user); + } + client.emit("event", presenceEvent); + }); + } + /* var i, j; // intercept the results and put them into our store diff --git a/lib/utils.js b/lib/utils.js index 955903fff..90416fb85 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -367,6 +367,69 @@ module.exports.runPolyfills = function() { return A; }; } + + // Array.prototype.forEach + // ======================================================== + // SOURCE: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach + // Production steps of ECMA-262, Edition 5, 15.4.4.18 + // Reference: http://es5.github.io/#x15.4.4.18 + if (!Array.prototype.forEach) { + + Array.prototype.forEach = function(callback, thisArg) { + + var T, k; + + if (this == null) { + throw new TypeError(' this is null or not defined'); + } + + // 1. Let O be the result of calling ToObject passing the |this| value as the argument. + var O = Object(this); + + // 2. Let lenValue be the result of calling the Get internal method of O with the argument "length". + // 3. Let len be ToUint32(lenValue). + var len = O.length >>> 0; + + // 4. If IsCallable(callback) is false, throw a TypeError exception. + // See: http://es5.github.com/#x9.11 + if (typeof callback !== "function") { + throw new TypeError(callback + ' is not a function'); + } + + // 5. If thisArg was supplied, let T be thisArg; else let T be undefined. + if (arguments.length > 1) { + T = thisArg; + } + + // 6. Let k be 0 + k = 0; + + // 7. Repeat, while k < len + while (k < len) { + + var kValue; + + // a. Let Pk be ToString(k). + // This is implicit for LHS operands of the in operator + // b. Let kPresent be the result of calling the HasProperty internal method of O with argument Pk. + // This step can be combined with c + // c. If kPresent is true, then + if (k in O) { + + // i. Let kValue be the result of calling the Get internal method of O with argument Pk. + kValue = O[k]; + + // ii. Call the Call internal method of callback with T as the this value and + // argument list containing kValue, k, and O. + callback.call(T, kValue, k, O); + } + // d. Increase k by 1. + k++; + } + // 8. return undefined + }; + } }; /** From 3d57b4ce6af0d78c733be4e3046df06eebdbb7bc Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 9 Dec 2015 16:41:36 +0000 Subject: [PATCH 09/32] Be paranoid on /sync processing --- lib/sync.js | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index 26c604b4e..77146b160 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -175,20 +175,30 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // } console.log("Got data %s", data); - // handle presence events (User objects) - if (data.presence && utils.isArray(data.presence.events)) { - data.presence.events.map(client.getEventMapper()).forEach(function(presenceEvent) { - var user = client.store.getUser(presenceEvent.getSender()); - if (user) { - user.setPresenceEvent(presenceEvent); - } - else { - user = createNewUser(client, presenceEvent.getSender()); - user.setPresenceEvent(presenceEvent); - client.store.storeUser(user); - } - client.emit("event", presenceEvent); - }); + // set the sync token NOW *before* processing the events. We do this so if something + // barfs on an event we can skip it rather than constantly polling with the same token. + client.store.setSyncToken(data.next_batch); + + try { + // handle presence events (User objects) + if (data.presence && utils.isArray(data.presence.events)) { + data.presence.events.map(client.getEventMapper()).forEach(function(presenceEvent) { + var user = client.store.getUser(presenceEvent.getSender()); + if (user) { + user.setPresenceEvent(presenceEvent); + } + else { + user = createNewUser(client, presenceEvent.getSender()); + user.setPresenceEvent(presenceEvent); + client.store.storeUser(user); + } + client.emit("event", presenceEvent); + }); + } + } + catch (e) { + console.error("Caught /sync error:"); + console.error(e); } /* @@ -292,11 +302,10 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { }); } + */ + - // assume success until we fail which may be 30+ secs */ - - client.store.setSyncToken(data.next_batch); - + // emit synced events if (!syncOptions.hasSyncedBefore) { updateSyncState(client, "PREPARED"); syncOptions.hasSyncedBefore = true; From 4c7e6807d2ad0f499942716ee47a36864dd6a126 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 10 Dec 2015 13:26:50 +0000 Subject: [PATCH 10/32] Parse invites from /sync --- lib/sync.js | 303 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 179 insertions(+), 124 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index 77146b160..eb0fb0068 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -41,7 +41,29 @@ function SyncApi(client, opts) { * @return {Room} */ SyncApi.prototype.createRoom = function(roomId) { - return createNewRoom(this.client, roomId); + var client = this.client; + var room = new Room(roomId, { + pendingEventOrdering: this.opts.pendingEventOrdering + }); + reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); + + // we need to also re-emit room state and room member events, so hook it up + // to the client now. We need to add a listener for RoomState.members in + // order to hook them correctly. (TODO: find a better way?) + reEmit(client, room.currentState, [ + "RoomState.events", "RoomState.members", "RoomState.newMember" + ]); + room.currentState.on("RoomState.newMember", function(event, state, member) { + member.user = client.getUser(member.userId); + reEmit( + client, member, + [ + "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", + "RoomMember.membership" + ] + ); + }); + return room; }; /** @@ -49,7 +71,27 @@ SyncApi.prototype.createRoom = function(roomId) { * @return {Promise} */ SyncApi.prototype.syncRoom = function(room) { - return _syncRoom(this.client, room); + var client = this.client; + var self = this; + if (client._syncingRooms[room.roomId]) { + return client._syncingRooms[room.roomId]; + } + var defer = q.defer(); + client._syncingRooms[room.roomId] = defer.promise; + client.roomInitialSync(room.roomId, this.opts.initialSyncLimit).done( + function(res) { + room.timeline = []; // blow away any previous messages. + self._processRoomEvents(room, res.state, res.messages); + room.recalculate(client.credentials.userId); + client.store.storeRoom(room); + client.emit("Room", room); + defer.resolve(room); + client._syncingRooms[room.roomId] = undefined; + }, function(err) { + defer.reject(err); + client._syncingRooms[room.roomId] = undefined; + }); + return defer.promise; }; /** @@ -173,7 +215,6 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // } // } // } - console.log("Got data %s", data); // set the sync token NOW *before* processing the events. We do this so if something // barfs on an event we can skip it rather than constantly polling with the same token. @@ -195,6 +236,46 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { client.emit("event", presenceEvent); }); } + + // the returned json structure is abit crap, so make it into a nicer form (array) after + // applying sanity to make sure we don't fail on missing keys (on the off chance) + var inviteRooms = []; + var joinRooms = []; + var leaveRooms = []; + + if (data.rooms) { + if (data.rooms.invite) { + inviteRooms = self._mapSyncResponseToRoomArray(data.rooms.invite); + } + if (data.rooms.join) { + joinRooms = self._mapSyncResponseToRoomArray(data.rooms.join); + } + if (data.rooms.leave) { + leaveRooms = self._mapSyncResponseToRoomArray(data.rooms.leave); + } + } + + // Handle invites + inviteRooms.forEach(function(inviteObj) { + var room = inviteObj.room; + var stateEvents = inviteObj.invite_state.events || []; + // add room_id back in f.e event + stateEvents = stateEvents.map(function(e) { + e.room_id = room.roomId; + return e; + }); + self._processRoomEvents(room, stateEvents); + if (inviteObj.isBrandNewRoom) { + room.recalculate(client.credentials.userId); + client.store.storeRoom(room); + client.emit("Room", room); + console.log("Storing %s %s", room.roomId, JSON.stringify(stateEvents)); + } + }); + + // Handle joins + + // Ignore leave rooms for now (TODO: Honour includeArchived opt) } catch (e) { console.error("Caught /sync error:"); @@ -322,10 +403,103 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { }); }; +SyncApi.prototype._mapSyncResponseToRoomArray = function(obj) { + // Maps { roomid: {stuff}, roomid: {stuff} } + // to + // [{stuff+Room+isBrandNewRoom}, {stuff+Room+isBrandNewRoom}] + var client = this.client; + var self = this; + return utils.keys(obj).map(function(roomId) { + var arrObj = obj[roomId]; + var room = client.store.getRoom(roomId); + var isBrandNewRoom = false; + if (!room) { + room = self.createRoom(roomId); + isBrandNewRoom = true; + } + arrObj.room = room; + arrObj.isBrandNewRoom = isBrandNewRoom; + return arrObj; + }); +} + +SyncApi.prototype._resolveInvites = function(room) { + if (!room || !this.opts.resolveInvitesToProfiles) { + return; + } + var client = this.client; + // For each invited room member we want to give them a displayname/avatar url + // if they have one (the m.room.member invites don't contain this). + room.getMembersWithMembership("invite").forEach(function(member) { + if (member._requestedProfileInfo) { + return; + } + member._requestedProfileInfo = true; + // try to get a cached copy first. + var user = client.getUser(member.userId); + var promise; + if (user) { + promise = q({ + avatar_url: user.avatarUrl, + displayname: user.displayName + }); + } + else { + promise = client.getProfileInfo(member.userId); + } + promise.done(function(info) { + // slightly naughty by doctoring the invite event but this means all + // the code paths remain the same between invite/join display name stuff + // which is a worthy trade-off for some minor pollution. + var inviteEvent = member.events.member; + if (inviteEvent.getContent().membership !== "invite") { + // between resolving and now they have since joined, so don't clobber + return; + } + inviteEvent.getContent().avatar_url = info.avatar_url; + inviteEvent.getContent().displayname = info.displayname; + member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners + }, function(err) { + // OH WELL. + }); + }); +} + +SyncApi.prototype._processRoomEvents = function(room, stateEventList, messageChunk) { + var client = this.client; + // "old" and "current" state are the same initially; they + // start diverging if the user paginates. + // We must deep copy otherwise membership changes in old state + // will leak through to current state! + var oldStateEvents = utils.map( + utils.deepCopy(stateEventList), client.getEventMapper() + ); + var stateEvents = utils.map(stateEventList, client.getEventMapper()); + room.oldState.setStateEvents(oldStateEvents); + room.currentState.setStateEvents(stateEvents); + + this._resolveInvites(room); + + // add events to the timeline *after* setting the state + // events so messages use the right display names. Initial sync + // returns messages in chronological order, so we need to reverse + // it to get most recent -> oldest. We need it in that order in + // order to diverge old/current state correctly. + room.addEventsToTimeline( + utils.map( + messageChunk ? messageChunk.chunk : [], + client.getEventMapper() + ).reverse(), true + ); + if (messageChunk) { + room.oldState.paginationToken = messageChunk.start; + } +} + /** * This is an internal method. * @param {Number=} attempt The attempt number - */ + */ /* SyncApi.prototype._pollForEvents = function(attempt) { var client = this.client; var self = this; @@ -472,8 +646,7 @@ SyncApi.prototype._pollForEvents = function(attempt) { }); updateSyncState(client, "ERROR", { error: err }); }); -}; - +}; */ function retryTimeMsForAttempt(attempt) { @@ -502,31 +675,6 @@ function createNewUser(client, userId) { return user; } -function createNewRoom(client, roomId) { - var room = new Room(roomId, { - pendingEventOrdering: client._config.pendingEventOrdering - }); - reEmit(client, room, ["Room.name", "Room.timeline", "Room.receipt", "Room.tags"]); - - // we need to also re-emit room state and room member events, so hook it up - // to the client now. We need to add a listener for RoomState.members in - // order to hook them correctly. (TODO: find a better way?) - reEmit(client, room.currentState, [ - "RoomState.events", "RoomState.members", "RoomState.newMember" - ]); - room.currentState.on("RoomState.newMember", function(event, state, member) { - member.user = client.getUser(member.userId); - reEmit( - client, member, - [ - "RoomMember.name", "RoomMember.typing", "RoomMember.powerLevel", - "RoomMember.membership" - ] - ); - }); - return room; -} - function reEmit(reEmitEntity, emittableEntity, eventNames) { utils.forEach(eventNames, function(eventName) { // setup a listener on the entity (the Room, User, etc) for this event @@ -545,98 +693,5 @@ function reEmit(reEmitEntity, emittableEntity, eventNames) { }); } -function _syncRoom(client, room) { - if (client._syncingRooms[room.roomId]) { - return client._syncingRooms[room.roomId]; - } - var defer = q.defer(); - client._syncingRooms[room.roomId] = defer.promise; - client.roomInitialSync(room.roomId, client._config.initialSyncLimit).done( - function(res) { - room.timeline = []; // blow away any previous messages. - _processRoomEvents(client, room, res.state, res.messages); - room.recalculate(client.credentials.userId); - client.store.storeRoom(room); - client.emit("Room", room); - defer.resolve(room); - client._syncingRooms[room.roomId] = undefined; - }, function(err) { - defer.reject(err); - client._syncingRooms[room.roomId] = undefined; - }); - return defer.promise; -} - -function _processRoomEvents(client, room, stateEventList, messageChunk) { - // "old" and "current" state are the same initially; they - // start diverging if the user paginates. - // We must deep copy otherwise membership changes in old state - // will leak through to current state! - var oldStateEvents = utils.map( - utils.deepCopy(stateEventList), client.getEventMapper() - ); - var stateEvents = utils.map(stateEventList, client.getEventMapper()); - room.oldState.setStateEvents(oldStateEvents); - room.currentState.setStateEvents(stateEvents); - - _resolveInvites(client, room); - - // add events to the timeline *after* setting the state - // events so messages use the right display names. Initial sync - // returns messages in chronological order, so we need to reverse - // it to get most recent -> oldest. We need it in that order in - // order to diverge old/current state correctly. - room.addEventsToTimeline( - utils.map( - messageChunk ? messageChunk.chunk : [], - client.getEventMapper() - ).reverse(), true - ); - if (messageChunk) { - room.oldState.paginationToken = messageChunk.start; - } -} - -function _resolveInvites(client, room) { - if (!room || !client._config.resolveInvitesToProfiles) { - return; - } - // For each invited room member we want to give them a displayname/avatar url - // if they have one (the m.room.member invites don't contain this). - room.getMembersWithMembership("invite").forEach(function(member) { - if (member._requestedProfileInfo) { - return; - } - member._requestedProfileInfo = true; - // try to get a cached copy first. - var user = client.getUser(member.userId); - var promise; - if (user) { - promise = q({ - avatar_url: user.avatarUrl, - displayname: user.displayName - }); - } - else { - promise = client.getProfileInfo(member.userId); - } - promise.done(function(info) { - // slightly naughty by doctoring the invite event but this means all - // the code paths remain the same between invite/join display name stuff - // which is a worthy trade-off for some minor pollution. - var inviteEvent = member.events.member; - if (inviteEvent.getContent().membership !== "invite") { - // between resolving and now they have since joined, so don't clobber - return; - } - inviteEvent.getContent().avatar_url = info.avatar_url; - inviteEvent.getContent().displayname = info.displayname; - member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners - }, function(err) { - // OH WELL. - }); - }); -} - /** */ module.exports = SyncApi; From 6af2197183c63bfa6930f40e85cf08cddedbbc02 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 10 Dec 2015 14:14:56 +0000 Subject: [PATCH 11/32] Process join rooms and add local timeouts to /sync This actually works now, though there's a number of teething issues which may be app-specific. That, and all the tests are broken. --- lib/client.js | 6 +- lib/sync.js | 351 +++++++++++++------------------------------------- 2 files changed, 88 insertions(+), 269 deletions(-) diff --git a/lib/client.js b/lib/client.js index d54140b67..4c2b036ce 100644 --- a/lib/client.js +++ b/lib/client.js @@ -138,9 +138,6 @@ function MatrixClient(opts) { userId: (opts.userId || null) }; this._http = new httpApi.MatrixHttpApi(httpOpts); - this._syncingRooms = { - // room_id: Promise - }; this.callList = { // callId: MatrixCall }; @@ -620,7 +617,8 @@ MatrixClient.prototype.joinRoom = function(roomIdOrAlias, opts, callback) { var syncApi = new SyncApi(self); var room = syncApi.createRoom(roomId); if (opts.syncRoom) { - return syncApi.syncRoom(room); + // v2 will do this for us + // return syncApi.syncRoom(room); } return q(room); }, function(err) { diff --git a/lib/sync.js b/lib/sync.js index eb0fb0068..a1a5e7614 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -66,34 +66,6 @@ SyncApi.prototype.createRoom = function(roomId) { return room; }; -/** - * @param {Room} room - * @return {Promise} - */ -SyncApi.prototype.syncRoom = function(room) { - var client = this.client; - var self = this; - if (client._syncingRooms[room.roomId]) { - return client._syncingRooms[room.roomId]; - } - var defer = q.defer(); - client._syncingRooms[room.roomId] = defer.promise; - client.roomInitialSync(room.roomId, this.opts.initialSyncLimit).done( - function(res) { - room.timeline = []; // blow away any previous messages. - self._processRoomEvents(room, res.state, res.messages); - room.recalculate(client.credentials.userId); - client.store.storeRoom(room); - client.emit("Room", room); - defer.resolve(room); - client._syncingRooms[room.roomId] = undefined; - }, function(err) { - defer.reject(err); - client._syncingRooms[room.roomId] = undefined; - }); - return defer.promise; -}; - /** * Main entry point */ @@ -182,13 +154,41 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { since: client.store.getSyncToken() || undefined // do not send 'null' }; + if (attempt > 1) { + // we think the connection is dead. If it comes back up, we won't know + // about it till /sync returns. If the timeout= is high, this could + // be a long time. Set it to 1 when doing retries. + qps.timeout = 1; + } + if (client._guestRooms && client._isGuest) { qps.room_id = JSON.stringify(client._guestRooms); } + // Set up local timer and error handler for retries. + function errHandler(err) { + console.error("/sync error (%s attempts): %s", attempt, err); + attempt += 1; + startSyncingRetryTimer(client, attempt, function() { + self._sync(syncOptions, attempt); + }); + updateSyncState(client, "ERROR", { error: err }); + } + var discardResult = false; + var timeoutObj = setTimeout(function() { + discardResult = true; + errHandler("Locally timed out waiting for a response"); + }, qps.timeout + (20 * 1000)); // 20s buffer + client._http.authedRequestWithPrefix( undefined, "GET", "/sync", qps, undefined, httpApi.PREFIX_V2_ALPHA ).done(function(data) { + if (discardResult) { + return; + } + else { + clearTimeout(timeoutObj); + } // data looks like: // { // next_batch: $token, @@ -220,6 +220,10 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // barfs on an event we can skip it rather than constantly polling with the same token. client.store.setSyncToken(data.next_batch); + // TODO-arch: + // - Each event we pass through needs to be emitted via 'event', can we do this in one place? + // - The isBrandNewRoom boilerplate is boilerplatey. + try { // handle presence events (User objects) if (data.presence && utils.isArray(data.presence.events)) { @@ -258,22 +262,33 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // Handle invites inviteRooms.forEach(function(inviteObj) { var room = inviteObj.room; - var stateEvents = inviteObj.invite_state.events || []; - // add room_id back in f.e event - stateEvents = stateEvents.map(function(e) { - e.room_id = room.roomId; - return e; - }); + var stateEvents = self._mapSyncEventsFormat(inviteObj.invite_state, room); self._processRoomEvents(room, stateEvents); if (inviteObj.isBrandNewRoom) { room.recalculate(client.credentials.userId); client.store.storeRoom(room); client.emit("Room", room); - console.log("Storing %s %s", room.roomId, JSON.stringify(stateEvents)); } + stateEvents.forEach(function(e) { client.emit("event", e); }); }); // Handle joins + joinRooms.forEach(function(joinObj) { + var room = joinObj.room; + var stateEvents = self._mapSyncEventsFormat(joinObj.state, room); + var timelineEvents = self._mapSyncEventsFormat(joinObj.timeline, room); + self._processRoomEvents( + room, stateEvents, timelineEvents, joinObj.timeline.prev_batch + ); + // TODO: Receipts, Typing, Tags from ephermeral + account_data + room.recalculate(client.credentials.userId); + if (joinObj.isBrandNewRoom) { + client.store.storeRoom(room); + client.emit("Room", room); + } + stateEvents.forEach(function(e) { client.emit("event", e); }); + timelineEvents.forEach(function(e) { client.emit("event", e); }); + }); // Ignore leave rooms for now (TODO: Honour includeArchived opt) } @@ -286,14 +301,6 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { var i, j; // intercept the results and put them into our store if (!(client.store instanceof StubStore)) { - utils.forEach( - utils.map(data.presence, client.getEventMapper()), - function(e) { - var user = createNewUser(client, e.getContent().user_id); - user.setPresenceEvent(e); - client.store.storeUser(user); - }); - // group receipts by room ID. var receiptsByRoom = {}; data.receipts = data.receipts || []; @@ -307,27 +314,6 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { ); for (i = 0; i < data.rooms.length; i++) { - var room = createNewRoom(client, data.rooms[i].room_id); - if (!data.rooms[i].state) { - data.rooms[i].state = []; - } - if (data.rooms[i].membership === "invite") { - var inviteEvent = data.rooms[i].invite; - if (!inviteEvent) { - // fallback for servers which don't serve the invite key yet - inviteEvent = { - event_id: "$fake_" + room.roomId, - content: { - membership: "invite" - }, - state_key: client.credentials.userId, - user_id: data.rooms[i].inviter, - room_id: room.roomId, - type: "m.room.member" - }; - } - data.rooms[i].state.push(inviteEvent); - } _processRoomEvents( client, room, data.rooms[i].state, data.rooms[i].messages @@ -359,30 +345,6 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { } } - if (data) { - var events = []; - for (i = 0; i < data.presence.length; i++) { - events.push(new MatrixEvent(data.presence[i])); - } - for (i = 0; i < data.rooms.length; i++) { - if (data.rooms[i].state) { - for (j = 0; j < data.rooms[i].state.length; j++) { - events.push(new MatrixEvent(data.rooms[i].state[j])); - } - } - if (data.rooms[i].messages) { - for (j = 0; j < data.rooms[i].messages.chunk.length; j++) { - events.push( - new MatrixEvent(data.rooms[i].messages.chunk[j]) - ); - } - } - } - utils.forEach(events, function(e) { - client.emit("event", e); - }); - } - */ @@ -393,14 +355,7 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { } updateSyncState(client, "SYNCING"); self._sync(syncOptions); - }, function(err) { - console.error("/sync error (%s attempts): %s", attempt, err); - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - self._sync(syncOptions, attempt); - }); - updateSyncState(client, "ERROR", { error: err }); - }); + }, errHandler); }; SyncApi.prototype._mapSyncResponseToRoomArray = function(obj) { @@ -423,6 +378,19 @@ SyncApi.prototype._mapSyncResponseToRoomArray = function(obj) { }); } +SyncApi.prototype._mapSyncEventsFormat = function(obj, room) { + if (!obj || !utils.isArray(obj.events)) { + return []; + } + var mapper = this.client.getEventMapper(); + return obj.events.map(function(e) { + if (room) { + e.room_id = room.roomId; + } + return mapper(e); + }); +}; + SyncApi.prototype._resolveInvites = function(room) { if (!room || !this.opts.resolveInvitesToProfiles) { return; @@ -465,190 +433,43 @@ SyncApi.prototype._resolveInvites = function(room) { }); } -SyncApi.prototype._processRoomEvents = function(room, stateEventList, messageChunk) { +/** + * @param {Room} room + * @param {MatrixEvent[]} stateEventList A list of state events. This is the state + * at the *START* of the timeline list if it is supplied. + * @param {MatrixEvent[]=} timelineEventList A list of timeline events. Lower index + * is earlier in time. Higher index is later. + * @param {string=} paginationToken + */ +SyncApi.prototype._processRoomEvents = function(room, stateEventList, timelineEventList, + paginationToken) { + timelineEventList = timelineEventList || []; var client = this.client; // "old" and "current" state are the same initially; they // start diverging if the user paginates. // We must deep copy otherwise membership changes in old state // will leak through to current state! var oldStateEvents = utils.map( - utils.deepCopy(stateEventList), client.getEventMapper() + utils.deepCopy( + stateEventList.map(function(mxEvent) { return mxEvent.event; }) + ), client.getEventMapper() ); - var stateEvents = utils.map(stateEventList, client.getEventMapper()); + var stateEvents = stateEventList; + + // set the state of the room to as it was before the timeline executes room.oldState.setStateEvents(oldStateEvents); room.currentState.setStateEvents(stateEvents); this._resolveInvites(room); - // add events to the timeline *after* setting the state - // events so messages use the right display names. Initial sync - // returns messages in chronological order, so we need to reverse - // it to get most recent -> oldest. We need it in that order in - // order to diverge old/current state correctly. - room.addEventsToTimeline( - utils.map( - messageChunk ? messageChunk.chunk : [], - client.getEventMapper() - ).reverse(), true - ); - if (messageChunk) { - room.oldState.paginationToken = messageChunk.start; + // execute the timeline events, this will begin to diverge the current state + // if the timeline has any state events in it. + room.addEventsToTimeline(timelineEventList); + if (paginationToken) { + room.oldState.paginationToken = paginationToken; } } -/** - * This is an internal method. - * @param {Number=} attempt The attempt number - */ /* -SyncApi.prototype._pollForEvents = function(attempt) { - var client = this.client; - var self = this; - - attempt = attempt || 1; - - if (!client.clientRunning) { - return; - } - var timeoutMs = client._config.pollTimeout; - if (attempt > 1) { - // we think the connection is dead. If it comes back up, we won't know - // about it till /events returns. If the timeout= is high, this could - // be a long time. Set it to 1 when doing retries. - timeoutMs = 1; - } - var discardResult = false; - var timeoutObj = setTimeout(function() { - discardResult = true; - console.error("/events request timed out."); - self._pollForEvents(); - }, timeoutMs + (20 * 1000)); // 20s buffer - - var queryParams = { - from: client.store.getSyncToken(), - timeout: timeoutMs - }; - if (client._guestRooms && client._isGuest) { - queryParams.room_id = client._guestRooms; - } - - client._http.authedRequest(undefined, "GET", "/events", queryParams).done( - function(data) { - if (discardResult) { - return; - } - else { - clearTimeout(timeoutObj); - } - - if (client._syncState !== "SYNCING") { - updateSyncState(client, "SYNCING"); - } - - try { - var events = []; - if (data) { - events = utils.map(data.chunk, client.getEventMapper()); - } - if (!(client.store instanceof StubStore)) { - var roomIdsWithNewInvites = {}; - // bucket events based on room. - var i = 0; - var roomIdToEvents = {}; - for (i = 0; i < events.length; i++) { - var roomId = events[i].getRoomId(); - // possible to have no room ID e.g. for presence events. - if (roomId) { - if (!roomIdToEvents[roomId]) { - roomIdToEvents[roomId] = []; - } - roomIdToEvents[roomId].push(events[i]); - if (events[i].getType() === "m.room.member" && - events[i].getContent().membership === "invite") { - roomIdsWithNewInvites[roomId] = true; - } - } - else if (events[i].getType() === "m.presence") { - var usr = client.store.getUser(events[i].getContent().user_id); - if (usr) { - usr.setPresenceEvent(events[i]); - } - else { - usr = createNewUser(client, events[i].getContent().user_id); - usr.setPresenceEvent(events[i]); - client.store.storeUser(usr); - } - } - } - - // add events to room - var roomIds = utils.keys(roomIdToEvents); - utils.forEach(roomIds, function(roomId) { - var room = client.store.getRoom(roomId); - var isBrandNewRoom = false; - if (!room) { - room = createNewRoom(client, roomId); - isBrandNewRoom = true; - } - - var wasJoined = room.hasMembershipState( - client.credentials.userId, "join" - ); - - room.addEvents(roomIdToEvents[roomId], "replace"); - room.recalculate(client.credentials.userId); - - // store the Room for things like invite events so developers - // can update the UI - if (isBrandNewRoom) { - client.store.storeRoom(room); - client.emit("Room", room); - } - - var justJoined = room.hasMembershipState( - client.credentials.userId, "join" - ); - - if (!wasJoined && justJoined) { - // we've just transitioned into a join state for this room, - // so sync state. - _syncRoom(client, room); - } - }); - - Object.keys(roomIdsWithNewInvites).forEach(function(inviteRoomId) { - _resolveInvites(client, client.store.getRoom(inviteRoomId)); - }); - } - if (data) { - client.store.setSyncToken(data.end); - utils.forEach(events, function(e) { - client.emit("event", e); - }); - } - } - catch (e) { - console.error("Event stream error:"); - console.error(e); - } - self._pollForEvents(); - }, function(err) { - console.error("/events error: %s", JSON.stringify(err)); - if (discardResult) { - return; - } - else { - clearTimeout(timeoutObj); - } - - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - self._pollForEvents(attempt); - }); - updateSyncState(client, "ERROR", { error: err }); - }); -}; */ - - function retryTimeMsForAttempt(attempt) { // 2,4,8,16,32,64,128,128,128,... seconds // max 2^7 secs = 2.1 mins From f08152a1d8676abf6df15fb3843c84a90f34a0ce Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 10 Dec 2015 14:27:21 +0000 Subject: [PATCH 12/32] Handle ephemeral and account_data events --- lib/sync.js | 61 ++++++++--------------------------------------------- 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index a1a5e7614..957a66a0e 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -277,10 +277,18 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { var room = joinObj.room; var stateEvents = self._mapSyncEventsFormat(joinObj.state, room); var timelineEvents = self._mapSyncEventsFormat(joinObj.timeline, room); + var ephemeralEvents = self._mapSyncEventsFormat(joinObj.ephemeral); + var accountDataEvents = self._mapSyncEventsFormat(joinObj.account_data); + + if (joinObj.timeline.limited) { + // nuke the timeline so we don't get holes + room.timeline = []; + } self._processRoomEvents( room, stateEvents, timelineEvents, joinObj.timeline.prev_batch ); - // TODO: Receipts, Typing, Tags from ephermeral + account_data + room.addEvents(ephemeralEvents); + room.addEvents(accountDataEvents); room.recalculate(client.credentials.userId); if (joinObj.isBrandNewRoom) { client.store.storeRoom(room); @@ -296,57 +304,6 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { console.error("Caught /sync error:"); console.error(e); } - - /* - var i, j; - // intercept the results and put them into our store - if (!(client.store instanceof StubStore)) { - // group receipts by room ID. - var receiptsByRoom = {}; - data.receipts = data.receipts || []; - utils.forEach(data.receipts.map(client.getEventMapper()), - function(receiptEvent) { - if (!receiptsByRoom[receiptEvent.getRoomId()]) { - receiptsByRoom[receiptEvent.getRoomId()] = []; - } - receiptsByRoom[receiptEvent.getRoomId()].push(receiptEvent); - } - ); - - for (i = 0; i < data.rooms.length; i++) { - - _processRoomEvents( - client, room, data.rooms[i].state, data.rooms[i].messages - ); - - var receipts = receiptsByRoom[room.roomId] || []; - for (j = 0; j < receipts.length; j++) { - room.addReceipt(receipts[j]); - } - - var privateUserData = data.rooms[i].account_data || []; - var privateUserDataEvents = - utils.map(privateUserData, client.getEventMapper()); - for (j = 0; j < privateUserDataEvents.length; j++) { - var event = privateUserDataEvents[j]; - if (event.getType() === "m.tag") { - room.addTags(event); - } - // XXX: unhandled private user data event - we should probably - // put it somewhere useful once the API has settled - } - - // cache the name/summary/etc prior to storage since we don't - // know how the store will serialise the Room. - room.recalculate(client.credentials.userId); - - client.store.storeRoom(room); - client.emit("Room", room); - } - } - - */ - // emit synced events if (!syncOptions.hasSyncedBefore) { From 8576ebce8fe687d809dc98ec6def5b1ab30c5d82 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 10 Dec 2015 14:57:13 +0000 Subject: [PATCH 13/32] Linting --- lib/store/memory.js | 6 +++--- lib/sync.js | 45 ++++++++++++++++++++++++++++++--------------- lib/utils.js | 13 ++++++++----- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/lib/store/memory.js b/lib/store/memory.js index 79cc25cae..5bdc3ef25 100644 --- a/lib/store/memory.js +++ b/lib/store/memory.js @@ -9,9 +9,9 @@ * Construct a new in-memory data store for the Matrix Client. * @constructor * @param {Object=} opts Config options - * @param {LocalStorage} opts.localStorage The local storage instance to persist some forms - * of data such as tokens. Rooms will NOT be stored. See {@link WebStorageStore} to - * persist rooms. + * @param {LocalStorage} opts.localStorage The local storage instance to persist + * some forms of data such as tokens. Rooms will NOT be stored. See + * {@link WebStorageStore} to persist rooms. */ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) { opts = opts || {}; diff --git a/lib/sync.js b/lib/sync.js index 957a66a0e..f6e47e435 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -9,11 +9,9 @@ * for HTTP and WS at some point. */ var q = require("q"); -var StubStore = require("./store/stub"); var User = require("./models/user"); var Room = require("./models/room"); var utils = require("./utils"); -var MatrixEvent = require("./models/event").MatrixEvent; var httpApi = require("./http-api"); var Filter = require("./filter"); @@ -216,18 +214,21 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // } // } - // set the sync token NOW *before* processing the events. We do this so if something - // barfs on an event we can skip it rather than constantly polling with the same token. + // set the sync token NOW *before* processing the events. We do this so + // if something barfs on an event we can skip it rather than constantly + // polling with the same token. client.store.setSyncToken(data.next_batch); // TODO-arch: - // - Each event we pass through needs to be emitted via 'event', can we do this in one place? + // - Each event we pass through needs to be emitted via 'event', can we + // do this in one place? // - The isBrandNewRoom boilerplate is boilerplatey. try { // handle presence events (User objects) if (data.presence && utils.isArray(data.presence.events)) { - data.presence.events.map(client.getEventMapper()).forEach(function(presenceEvent) { + data.presence.events.map(client.getEventMapper()).forEach( + function(presenceEvent) { var user = client.store.getUser(presenceEvent.getSender()); if (user) { user.setPresenceEvent(presenceEvent); @@ -241,8 +242,9 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { }); } - // the returned json structure is abit crap, so make it into a nicer form (array) after - // applying sanity to make sure we don't fail on missing keys (on the off chance) + // the returned json structure is abit crap, so make it into a + // nicer form (array) after applying sanity to make sure we don't fail + // on missing keys (on the off chance) var inviteRooms = []; var joinRooms = []; var leaveRooms = []; @@ -304,7 +306,7 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { console.error("Caught /sync error:"); console.error(e); } - + // emit synced events if (!syncOptions.hasSyncedBefore) { updateSyncState(client, "PREPARED"); @@ -315,6 +317,10 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { }, errHandler); }; +/** + * @param {Object} obj + * @return {Object[]} + */ SyncApi.prototype._mapSyncResponseToRoomArray = function(obj) { // Maps { roomid: {stuff}, roomid: {stuff} } // to @@ -333,8 +339,13 @@ SyncApi.prototype._mapSyncResponseToRoomArray = function(obj) { arrObj.isBrandNewRoom = isBrandNewRoom; return arrObj; }); -} +}; +/** + * @param {Object} obj + * @param {Room} room + * @return {MatrixEvent[]} + */ SyncApi.prototype._mapSyncEventsFormat = function(obj, room) { if (!obj || !utils.isArray(obj.events)) { return []; @@ -348,6 +359,9 @@ SyncApi.prototype._mapSyncEventsFormat = function(obj, room) { }); }; +/** + * @param {Room} room + */ SyncApi.prototype._resolveInvites = function(room) { if (!room || !this.opts.resolveInvitesToProfiles) { return; @@ -383,12 +397,13 @@ SyncApi.prototype._resolveInvites = function(room) { } inviteEvent.getContent().avatar_url = info.avatar_url; inviteEvent.getContent().displayname = info.displayname; - member.setMembershipEvent(inviteEvent, room.currentState); // fire listeners + // fire listeners + member.setMembershipEvent(inviteEvent, room.currentState); }, function(err) { // OH WELL. }); }); -} +}; /** * @param {Room} room @@ -398,8 +413,8 @@ SyncApi.prototype._resolveInvites = function(room) { * is earlier in time. Higher index is later. * @param {string=} paginationToken */ -SyncApi.prototype._processRoomEvents = function(room, stateEventList, timelineEventList, - paginationToken) { +SyncApi.prototype._processRoomEvents = function(room, stateEventList, + timelineEventList, paginationToken) { timelineEventList = timelineEventList || []; var client = this.client; // "old" and "current" state are the same initially; they @@ -425,7 +440,7 @@ SyncApi.prototype._processRoomEvents = function(room, stateEventList, timelineEv if (paginationToken) { room.oldState.paginationToken = paginationToken; } -} +}; function retryTimeMsForAttempt(attempt) { // 2,4,8,16,32,64,128,128,128,... seconds diff --git a/lib/utils.js b/lib/utils.js index 90416fb85..32741f2f6 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -371,7 +371,7 @@ module.exports.runPolyfills = function() { // Array.prototype.forEach // ======================================================== // SOURCE: - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach // Production steps of ECMA-262, Edition 5, 15.4.4.18 // Reference: http://es5.github.io/#x15.4.4.18 if (!Array.prototype.forEach) { @@ -380,14 +380,15 @@ module.exports.runPolyfills = function() { var T, k; - if (this == null) { + if (this === null || this === undefined) { throw new TypeError(' this is null or not defined'); } // 1. Let O be the result of calling ToObject passing the |this| value as the argument. var O = Object(this); - // 2. Let lenValue be the result of calling the Get internal method of O with the argument "length". + // 2. Let lenValue be the result of calling the Get internal method of O with the + // argument "length". // 3. Let len be ToUint32(lenValue). var len = O.length >>> 0; @@ -412,12 +413,14 @@ module.exports.runPolyfills = function() { // a. Let Pk be ToString(k). // This is implicit for LHS operands of the in operator - // b. Let kPresent be the result of calling the HasProperty internal method of O with argument Pk. + // b. Let kPresent be the result of calling the HasProperty internal + // method of O with + // argument Pk. // This step can be combined with c // c. If kPresent is true, then if (k in O) { - // i. Let kValue be the result of calling the Get internal method of O with argument Pk. + // i. Let kValue be the result of calling the Get internal method of O with argument Pk kValue = O[k]; // ii. Call the Call internal method of callback with T as the this value and From d6ba39f292eb58d78bbb78a3cd18ede6591982c0 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Thu, 10 Dec 2015 15:01:39 +0000 Subject: [PATCH 14/32] More linting --- lib/store/memory.js | 2 +- lib/sync.js | 1 + lib/utils.js | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/store/memory.js b/lib/store/memory.js index 5bdc3ef25..b7af1bfd8 100644 --- a/lib/store/memory.js +++ b/lib/store/memory.js @@ -159,7 +159,7 @@ module.exports.MatrixInMemoryStore.prototype = { try { return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName); } - catch(e) {} + catch (e) {} return null; }, diff --git a/lib/sync.js b/lib/sync.js index f6e47e435..6ca40e87a 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -22,6 +22,7 @@ var FILTER_SYNC = "FILTER_SYNC"; * Construct an entity which is able to sync with a homeserver. * @constructor * @param {MatrixClient} client The matrix client instance to use. + * @param {Object} opts Config options */ function SyncApi(client, opts) { this.client = client; diff --git a/lib/utils.js b/lib/utils.js index 32741f2f6..f58a6533b 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -384,7 +384,8 @@ module.exports.runPolyfills = function() { throw new TypeError(' this is null or not defined'); } - // 1. Let O be the result of calling ToObject passing the |this| value as the argument. + // 1. Let O be the result of calling ToObject passing the |this| value as the + // argument. var O = Object(this); // 2. Let lenValue be the result of calling the Get internal method of O with the @@ -420,7 +421,8 @@ module.exports.runPolyfills = function() { // c. If kPresent is true, then if (k in O) { - // i. Let kValue be the result of calling the Get internal method of O with argument Pk + // i. Let kValue be the result of calling the Get internal method of O with + // argument Pk kValue = O[k]; // ii. Call the Call internal method of callback with T as the this value and From 26d9fed537cdb2044f7de26614b978d0bdcd64ac Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 11:07:31 +0000 Subject: [PATCH 15/32] Fix MatrixClient unit tests --- lib/client.js | 6 +- lib/sync.js | 2 +- spec/unit/matrix-client.spec.js | 165 +++++++++++++++++++------------- 3 files changed, 102 insertions(+), 71 deletions(-) diff --git a/lib/client.js b/lib/client.js index 4c2b036ce..5091a5828 100644 --- a/lib/client.js +++ b/lib/client.js @@ -2502,7 +2502,7 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED; * a state of SYNCING. This is the equivalent of "syncComplete" in the * previous API. *
  • SYNCING : The client is currently polling for new events from the server. - * The client may fire this before or after processing latest events from a sync.
    • + * This will be called after processing latest events from a sync. *
  • ERROR : The client has had a problem syncing with the server. If this is * called before PREPARED then there was a problem performing the initial * sync. If this is called after PREPARED then there was a problem polling @@ -2523,7 +2523,7 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED; * Transitions: *
      *
    • null -> PREPARED : Occurs when the initial sync is completed - * first time. + * first time. This involves setting up filters and obtaining push rules. *
    • null -> ERROR : Occurs when the initial sync failed first time. *
    • ERROR -> PREPARED : Occurs when the initial sync succeeds * after previously failing. @@ -2535,6 +2535,8 @@ module.exports.CRYPTO_ENABLED = CRYPTO_ENABLED; * live update after having previously failed. *
    • ERROR -> ERROR : Occurs when the client has failed to sync * for a second time or more.
      • + *
    • SYNCING -> SYNCING : Occurs when the client has performed a live + * update. This is called after processing.
      • *
    * * @event module:client~MatrixClient#"sync" diff --git a/lib/sync.js b/lib/sync.js index 6ca40e87a..da0f1f4c1 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -161,7 +161,7 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { } if (client._guestRooms && client._isGuest) { - qps.room_id = JSON.stringify(client._guestRooms); + qps.room_id = client._guestRooms; } // Set up local timer and error handler for retries. diff --git a/spec/unit/matrix-client.spec.js b/spec/unit/matrix-client.spec.js index 5d5093e7a..16d3330b4 100644 --- a/spec/unit/matrix-client.spec.js +++ b/spec/unit/matrix-client.spec.js @@ -9,20 +9,30 @@ describe("MatrixClient", function() { var identityServerUrl = "https://identity.server"; var client, store, scheduler; - var initialSyncData = { - end: "s_5_3", - presence: [], - rooms: [] - }; - - var eventData = { - start: "s_START", - end: "s_END", - chunk: [] - }; - var PUSH_RULES_RESPONSE = { - method: "GET", path: "/pushrules/", data: {} + method: "GET", + path: "/pushrules/", + data: {} + }; + + var FILTER_PATH = "/user/" + encodeURIComponent(userId) + "/filter"; + + var FILTER_RESPONSE = { + method: "POST", + path: FILTER_PATH, + data: { filter_id: "f1lt3r" } + }; + + var SYNC_DATA = { + next_batch: "s_5_3", + presence: { events: [] }, + rooms: {} + }; + + var SYNC_RESPONSE = { + method: "GET", + path: "/sync", + data: SYNC_DATA }; var httpLookups = [ @@ -107,7 +117,8 @@ describe("MatrixClient", function() { ]); store = jasmine.createSpyObj("store", [ "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback", - "setSyncToken", "storeEvents", "storeRoom", "storeUser" + "setSyncToken", "storeEvents", "storeRoom", "storeUser", + "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter" ]); client = new MatrixClient({ baseUrl: "https://my.home.server", @@ -130,9 +141,8 @@ describe("MatrixClient", function() { pendingLookup = null; httpLookups = []; httpLookups.push(PUSH_RULES_RESPONSE); - httpLookups.push({ - method: "GET", path: "/initialSync", data: initialSyncData - }); + httpLookups.push(FILTER_RESPONSE); + httpLookups.push(SYNC_RESPONSE); }); afterEach(function() { @@ -156,9 +166,10 @@ describe("MatrixClient", function() { }); it("should return the same sync state as emitted sync events", function(done) { - client.on("sync", function(state) { + client.on("sync", function syncListener(state) { expect(state).toEqual(client.getSyncState()); if (state === "SYNCING") { + client.removeListener("sync", syncListener); done(); } }); @@ -172,40 +183,46 @@ describe("MatrixClient", function() { expect(client.retryImmediately()).toBe(false); }); - it("should work on /initialSync", function(done) { + it("should work on /filter", function(done) { httpLookups = []; httpLookups.push(PUSH_RULES_RESPONSE); httpLookups.push({ - method: "GET", path: "/initialSync", error: { errcode: "NOPE_NOPE_NOPE" } + method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" } }); httpLookups.push({ - method: "GET", path: "/initialSync", error: { errcode: "NOPE_NOPE_NOPE" } + method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" } }); - client.on("sync", function(state) { + client.on("sync", function syncListener(state) { if (state === "ERROR" && httpLookups.length > 0) { expect(httpLookups.length).toEqual(1); expect(client.retryImmediately()).toBe(true); expect(httpLookups.length).toEqual(0); + client.removeListener("sync", syncListener); done(); } }); client.startClient(); }); - it("should work on /events", function(done) { + it("should work on /sync", function(done) { httpLookups.push({ - method: "GET", path: "/events", error: { errcode: "NOPE_NOPE_NOPE" } + method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" } }); httpLookups.push({ - method: "GET", path: "/events", data: eventData + method: "GET", path: "/sync", data: SYNC_DATA }); - client.on("sync", function(state) { + client.on("sync", function syncListener(state) { if (state === "ERROR" && httpLookups.length > 0) { expect(httpLookups.length).toEqual(1); - expect(client.retryImmediately()).toBe(true); - expect(httpLookups.length).toEqual(0); + expect(client.retryImmediately()).toBe( + true, "retryImmediately returned false" + ); + expect(httpLookups.length).toEqual( + 0, "more httpLookups remaining than expected" + ); + client.removeListener("sync", syncListener); done(); } }); @@ -221,11 +238,12 @@ describe("MatrixClient", function() { method: "GET", path: "/pushrules/", error: { errcode: "NOPE_NOPE_NOPE" } }); - client.on("sync", function(state) { + client.on("sync", function syncListener(state) { if (state === "ERROR" && httpLookups.length > 0) { expect(httpLookups.length).toEqual(1); expect(client.retryImmediately()).toBe(true); expect(httpLookups.length).toEqual(0); + client.removeListener("sync", syncListener); done(); } }); @@ -234,10 +252,9 @@ describe("MatrixClient", function() { }); describe("emitted sync events", function() { - var expectedStates; - function syncChecker(done) { - return function(state, old) { + function syncChecker(expectedStates, done) { + return function syncListener(state, old) { var expected = expectedStates.shift(); console.log( "'sync' curr=%s old=%s EXPECT=%s", state, old, expected @@ -249,6 +266,7 @@ describe("MatrixClient", function() { expect(state).toEqual(expected[0]); expect(old).toEqual(expected[1]); if (expectedStates.length === 0) { + client.removeListener("sync", syncListener); done(); } // standard retry time is 4s @@ -256,94 +274,107 @@ describe("MatrixClient", function() { }; } - beforeEach(function() { - expectedStates = [ - // [current, old] - ]; - }); - - it("should transition null -> PREPARED after /initialSync", function(done) { + it("should transition null -> PREPARED after the first /sync", function(done) { + var expectedStates = []; expectedStates.push(["PREPARED", null]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition null -> ERROR after a failed /initialSync", function(done) { + it("should transition null -> ERROR after a failed /filter", function(done) { + var expectedStates = []; httpLookups = []; httpLookups.push(PUSH_RULES_RESPONSE); httpLookups.push({ - method: "GET", path: "/initialSync", error: { errcode: "NOPE_NOPE_NOPE" } + method: "POST", path: FILTER_PATH, error: { errcode: "NOPE_NOPE_NOPE" } }); expectedStates.push(["ERROR", null]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition ERROR -> PREPARED after /initialSync if prev failed", + it("should transition ERROR -> PREPARED after /sync if prev failed", function(done) { + var expectedStates = []; httpLookups = []; httpLookups.push(PUSH_RULES_RESPONSE); + httpLookups.push(FILTER_RESPONSE); httpLookups.push({ - method: "GET", path: "/initialSync", error: { errcode: "NOPE_NOPE_NOPE" } + method: "GET", path: "/sync", error: { errcode: "NOPE_NOPE_NOPE" } }); httpLookups.push({ - method: "GET", path: "/initialSync", data: initialSyncData + method: "GET", path: "/sync", data: SYNC_DATA }); expectedStates.push(["ERROR", null]); expectedStates.push(["PREPARED", "ERROR"]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition PREPARED -> SYNCING after /initialSync", function(done) { + it("should transition PREPARED -> SYNCING after /sync", function(done) { + var expectedStates = []; expectedStates.push(["PREPARED", null]); expectedStates.push(["SYNCING", "PREPARED"]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition SYNCING -> ERROR after a failed /events", function(done) { + it("should transition SYNCING -> ERROR after a failed /sync", function(done) { + var expectedStates = []; httpLookups.push({ - method: "GET", path: "/events", error: { errcode: "NONONONONO" } + method: "GET", path: "/sync", error: { errcode: "NONONONONO" } }); expectedStates.push(["PREPARED", null]); expectedStates.push(["SYNCING", "PREPARED"]); expectedStates.push(["ERROR", "SYNCING"]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition ERROR -> SYNCING after /events if prev failed", + xit("should transition ERROR -> SYNCING after /sync if prev failed", function(done) { + var expectedStates = []; httpLookups.push({ - method: "GET", path: "/events", error: { errcode: "NONONONONO" } - }); - httpLookups.push({ - method: "GET", path: "/events", data: eventData + method: "GET", path: "/sync", error: { errcode: "NONONONONO" } }); + httpLookups.push(SYNC_RESPONSE); expectedStates.push(["PREPARED", null]); expectedStates.push(["SYNCING", "PREPARED"]); expectedStates.push(["ERROR", "SYNCING"]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); - it("should transition ERROR -> ERROR if multiple /events fails", function(done) { + it("should transition SYNCING -> SYNCING on subsequent /sync successes", + function(done) { + var expectedStates = []; + httpLookups.push(SYNC_RESPONSE); + httpLookups.push(SYNC_RESPONSE); + + expectedStates.push(["PREPARED", null]); + expectedStates.push(["SYNCING", "PREPARED"]); + expectedStates.push(["SYNCING", "SYNCING"]); + client.on("sync", syncChecker(expectedStates, done)); + client.startClient(); + }); + + it("should transition ERROR -> ERROR if multiple /sync fails", function(done) { + var expectedStates = []; httpLookups.push({ - method: "GET", path: "/events", error: { errcode: "NONONONONO" } + method: "GET", path: "/sync", error: { errcode: "NONONONONO" } }); httpLookups.push({ - method: "GET", path: "/events", error: { errcode: "NONONONONO" } + method: "GET", path: "/sync", error: { errcode: "NONONONONO" } }); expectedStates.push(["PREPARED", null]); expectedStates.push(["SYNCING", "PREPARED"]); expectedStates.push(["ERROR", "SYNCING"]); expectedStates.push(["ERROR", "ERROR"]); - client.on("sync", syncChecker(done)); + client.on("sync", syncChecker(expectedStates, done)); client.startClient(); }); }); @@ -373,15 +404,13 @@ describe("MatrixClient", function() { "!foo:bar", "!baz:bar" ]; - it("should be set via setGuestRooms and used in /events calls", function(done) { + it("should be set via setGuestRooms and used in /sync calls", function(done) { httpLookups = []; // no /pushrules - httpLookups.push({ - method: "GET", path: "/initialSync", data: initialSyncData - }); + httpLookups.push(FILTER_RESPONSE); httpLookups.push({ method: "GET", - path: "/events", - data: eventData, + path: "/sync", + data: SYNC_DATA, expectQueryParams: { room_id: roomIds }, From b219836b3eacfa515e7b41b04cd4262ced5ff608 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 12:53:26 +0000 Subject: [PATCH 16/32] Fix a bunch of integration tests --- lib/store/stub.js | 9 + lib/sync.js | 2 + .../integ/matrix-client-event-emitter.spec.js | 184 ++++++------ spec/integ/matrix-client-opts.spec.js | 86 +++--- .../integ/matrix-client-room-timeline.spec.js | 262 +++++++++++------- 5 files changed, 309 insertions(+), 234 deletions(-) diff --git a/lib/store/stub.js b/lib/store/stub.js index 1c835eca9..4cdc1a93e 100644 --- a/lib/store/stub.js +++ b/lib/store/stub.js @@ -122,6 +122,15 @@ StubStore.prototype = { */ getFilterIdByName: function(filterName) { return null; + }, + + /** + * Set a filter name to ID mapping. + * @param {string} filterName + * @param {string} filterId + */ + setFilterIdByName: function(filterName, filterId) { + } // TODO diff --git a/lib/sync.js b/lib/sync.js index da0f1f4c1..52371fc7c 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -299,6 +299,8 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { } stateEvents.forEach(function(e) { client.emit("event", e); }); timelineEvents.forEach(function(e) { client.emit("event", e); }); + ephemeralEvents.forEach(function(e) { client.emit("event", e); }); + accountDataEvents.forEach(function(e) { client.emit("event", e); }); }); // Ignore leave rooms for now (TODO: Honour includeArchived opt) diff --git a/spec/integ/matrix-client-event-emitter.spec.js b/spec/integ/matrix-client-event-emitter.spec.js index a6c04f36f..e249ce149 100644 --- a/spec/integ/matrix-client-event-emitter.spec.js +++ b/spec/integ/matrix-client-event-emitter.spec.js @@ -19,6 +19,7 @@ describe("MatrixClient events", function() { accessToken: selfAccessToken }); httpBackend.when("GET", "/pushrules").respond(200, {}); + httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" }); }); afterEach(function() { @@ -26,108 +27,113 @@ describe("MatrixClient events", function() { }); describe("emissions", function() { - var initialSync = { - end: "s_5_3", - presence: [{ - event_id: "$wefiuewh:bar", - type: "m.presence", - content: { - user_id: "@foo:bar", - displayname: "Foo Bar", - presence: "online" - } - }], - rooms: [{ - room_id: "!erufh:bar", - membership: "join", - messages: { - start: "s", - end: "t", - chunk: [ - utils.mkMessage({ - room: "!erufh:bar", user: "@foo:bar", msg: "hmmm" - }) - ] - }, - state: [ - utils.mkMembership({ - room: "!erufh:bar", mship: "join", user: "@foo:bar" - }), - utils.mkEvent({ - type: "m.room.create", room: "!erufh:bar", user: "@foo:bar", - content: { - creator: "@foo:bar" - } + var SYNC_DATA = { + next_batch: "s_5_3", + presence: { + events: [ + utils.mkPresence({ + user: "@foo:bar", name: "Foo Bar", presence: "online" }) ] - }] - }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [ - utils.mkMessage({ - room: "!erufh:bar", user: "@foo:bar", msg: "ello ello" - }), - utils.mkMessage({ - room: "!erufh:bar", user: "@foo:bar", msg: ":D" - }), - utils.mkEvent({ - type: "m.typing", room: "!erufh:bar", content: { - user_ids: ["@foo:bar"] + }, + rooms: { + join: { + "!erufh:bar": { + timeline: { + events: [ + utils.mkMessage({ + room: "!erufh:bar", user: "@foo:bar", msg: "hmmm" + }) + ], + prev_batch: "s" + }, + state: { + events: [ + utils.mkMembership({ + room: "!erufh:bar", mship: "join", user: "@foo:bar" + }), + utils.mkEvent({ + type: "m.room.create", room: "!erufh:bar", user: "@foo:bar", + content: { + creator: "@foo:bar" + } + }) + ] + } } - }) - ] + } + } + }; + var NEXT_SYNC_DATA = { + next_batch: "e_6_7", + rooms: { + join: { + "!erufh:bar": { + timeline: { + events: [ + utils.mkMessage({ + room: "!erufh:bar", user: "@foo:bar", msg: "ello ello" + }), + utils.mkMessage({ + room: "!erufh:bar", user: "@foo:bar", msg: ":D" + }), + ] + }, + ephemeral: { + events: [ + utils.mkEvent({ + type: "m.typing", room: "!erufh:bar", content: { + user_ids: ["@foo:bar"] + } + }) + ] + } + } + } + } }; - it("should emit events from both /initialSync and /events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + it("should emit events from both the first and subsequent /sync calls", + function(done) { + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); + + var expectedEvents = []; + expectedEvents = expectedEvents.concat( + SYNC_DATA.presence.events, + SYNC_DATA.rooms.join["!erufh:bar"].timeline.events, + SYNC_DATA.rooms.join["!erufh:bar"].state.events, + NEXT_SYNC_DATA.rooms.join["!erufh:bar"].timeline.events, + NEXT_SYNC_DATA.rooms.join["!erufh:bar"].ephemeral.events + ); + - // initial sync events are unordered, so make an array of the types - // that should be emitted and we'll just pick them off one by one, - // so long as this is emptied we're good. - var initialSyncEventTypes = [ - "m.presence", "m.room.member", "m.room.message", "m.room.create" - ]; var chunkIndex = 0; client.on("event", function(event) { - if (initialSyncEventTypes.length === 0) { - if (chunkIndex + 1 >= eventData.chunk.length) { - return; + var found = false; + for (var i = 0; i < expectedEvents.length; i++) { + if (expectedEvents[i].event_id === event.getId()) { + expectedEvents.splice(i, 1); + found = true; + break; } - // this should be /events now - expect(eventData.chunk[chunkIndex].event_id).toEqual( - event.getId() - ); - chunkIndex++; - return; - } - var index = initialSyncEventTypes.indexOf(event.getType()); - expect(index).not.toEqual( - -1, "Unexpected event type: " + event.getType() - ); - if (index >= 0) { - initialSyncEventTypes.splice(index, 1); } + expect(found).toBe(true, "Unexpected 'event' emitted: " + event.getType()); }); client.startClient(); httpBackend.flush().done(function() { - expect(initialSyncEventTypes.length).toEqual( - 0, "Failed to see all events from /initialSync" - ); - expect(chunkIndex + 1).toEqual( - eventData.chunk.length, "Failed to see all events from /events" + expect(expectedEvents.length).toEqual( + 0, "Failed to see all events from /sync calls" ); done(); }); }); it("should emit User events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); var fired = false; client.on("User.presence", function(event, user) { fired = true; @@ -135,9 +141,9 @@ describe("MatrixClient events", function() { expect(event).toBeDefined(); if (!user || !event) { return; } - expect(event.event).toEqual(initialSync.presence[0]); + expect(event.event).toEqual(SYNC_DATA.presence.events[0]); expect(user.presence).toEqual( - initialSync.presence[0].content.presence + SYNC_DATA.presence.events[0].content.presence ); }); client.startClient(); @@ -149,8 +155,8 @@ describe("MatrixClient events", function() { }); it("should emit Room events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); var roomInvokeCount = 0; var roomNameInvokeCount = 0; var timelineFireCount = 0; @@ -183,8 +189,8 @@ describe("MatrixClient events", function() { }); it("should emit RoomState events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); var roomStateEventTypes = [ "m.room.member", "m.room.create" @@ -232,8 +238,8 @@ describe("MatrixClient events", function() { }); it("should emit RoomMember events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); var typingInvokeCount = 0; var powerLevelInvokeCount = 0; diff --git a/spec/integ/matrix-client-opts.spec.js b/spec/integ/matrix-client-opts.spec.js index d8c010c78..347e3acd3 100644 --- a/spec/integ/matrix-client-opts.spec.js +++ b/spec/integ/matrix-client-opts.spec.js @@ -11,47 +11,45 @@ describe("MatrixClient opts", function() { var userB = "@bob:localhost"; var accessToken = "aseukfgwef"; var roomId = "!foo:bar"; - var eventData = { - chunk: [], - start: "s", - end: "e" - }; - var initialSync = { - end: "s_5_3", - presence: [], - rooms: [{ - membership: "join", - room_id: roomId, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ - utils.mkMessage({ - room: roomId, user: userB, msg: "hello" - }) - ] - }, - state: [ - utils.mkEvent({ - type: "m.room.name", room: roomId, user: userB, - content: { - name: "Old room name" + var syncData = { + next_batch: "s_5_3", + presence: {}, + rooms: { + join: { + "!foo:bar": { // roomId + timeline: { + events: [ + utils.mkMessage({ + room: roomId, user: userB, msg: "hello" + }) + ], + prev_batch: "f_1_1" + }, + state: { + events: [ + utils.mkEvent({ + type: "m.room.name", room: roomId, user: userB, + content: { + name: "Old room name" + } + }), + utils.mkMembership({ + room: roomId, mship: "join", user: userB, name: "Bob" + }), + utils.mkMembership({ + room: roomId, mship: "join", user: userId, name: "Alice" + }), + utils.mkEvent({ + type: "m.room.create", room: roomId, user: userId, + content: { + creator: userId + } + }) + ] } - }), - utils.mkMembership({ - room: roomId, mship: "join", user: userB, name: "Bob" - }), - utils.mkMembership({ - room: roomId, mship: "join", user: userId, name: "Alice" - }), - utils.mkEvent({ - type: "m.room.create", room: roomId, user: userId, - content: { - creator: userId - } - }) - ] - }] + } + } + } }; beforeEach(function() { @@ -101,13 +99,13 @@ describe("MatrixClient opts", function() { ); }); httpBackend.when("GET", "/pushrules").respond(200, {}); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("POST", "/filter").respond(200, { filter_id: "foo" }); + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient(); httpBackend.flush("/pushrules", 1).then(function() { - return httpBackend.flush("/initialSync", 1); + return httpBackend.flush("/filter", 1); }).then(function() { - return httpBackend.flush("/events", 1); + return httpBackend.flush("/sync", 1); }).done(function() { expect(expectedEventTypes.length).toEqual( 0, "Expected to see event types: " + expectedEventTypes diff --git a/spec/integ/matrix-client-room-timeline.spec.js b/spec/integ/matrix-client-room-timeline.spec.js index 58d32c011..59a45704c 100644 --- a/spec/integ/matrix-client-room-timeline.spec.js +++ b/spec/integ/matrix-client-room-timeline.spec.js @@ -12,45 +12,93 @@ describe("MatrixClient room timelines", function() { var accessToken = "aseukfgwef"; var roomId = "!foo:bar"; var otherUserId = "@bob:localhost"; - var eventData; - var initialSync = { - end: "s_5_3", - presence: [], - rooms: [{ - membership: "join", - room_id: roomId, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ - utils.mkMessage({ - room: roomId, user: otherUserId, msg: "hello" - }) - ] - }, - state: [ - utils.mkEvent({ - type: "m.room.name", room: roomId, user: otherUserId, - content: { - name: "Old room name" + var USER_MEMBERSHIP_EVENT = utils.mkMembership({ + room: roomId, mship: "join", user: userId, name: userName + }); + var ROOM_NAME_EVENT = utils.mkEvent({ + type: "m.room.name", room: roomId, user: otherUserId, + content: { + name: "Old room name" + } + }); + var NEXT_SYNC_DATA; + var SYNC_DATA = { + next_batch: "s_5_3", + rooms: { + join: { + "!foo:bar": { // roomId + timeline: { + events: [ + utils.mkMessage({ + room: roomId, user: otherUserId, msg: "hello" + }) + ], + prev_batch: "f_1_1" + }, + state: { + events: [ + ROOM_NAME_EVENT, + utils.mkMembership({ + room: roomId, mship: "join", user: otherUserId, name: "Bob" + }), + USER_MEMBERSHIP_EVENT, + utils.mkEvent({ + type: "m.room.create", room: roomId, user: userId, + content: { + creator: userId + } + }) + ] } - }), - utils.mkMembership({ - room: roomId, mship: "join", user: otherUserId, name: "Bob" - }), - utils.mkMembership({ - room: roomId, mship: "join", user: userId, name: userName - }), - utils.mkEvent({ - type: "m.room.create", room: roomId, user: userId, - content: { - creator: userId - } - }) - ] - }] + } + } + } }; + function setNextSyncData(events) { + events = events || []; + NEXT_SYNC_DATA = { + next_batch: "n", + presence: { events: [] }, + rooms: { + invite: {}, + join: { + "!foo:bar": { + timeline: { events: [] }, + state: { events: [] }, + ephemeral: { events: [] } + } + }, + leave: {} + } + }; + events.forEach(function(e) { + if (e.room_id !== roomId) { + throw new Error("setNextSyncData only works with one room id"); + } + if (e.state_key) { + if (e.__prev_event === undefined) { + throw new Error( + "setNextSyncData needs the prev state set to '__prev_event' " + + "for " + e.type + ); + } + if (e.__prev_event !== null) { + // push the previous state for this event type + NEXT_SYNC_DATA.rooms.join[roomId].state.events.push(e.__prev_event); + } + // push the current + NEXT_SYNC_DATA.rooms.join[roomId].timeline.events.push(e); + } + else if (["m.typing", "m.receipt"].indexOf(e.type) !== -1) { + NEXT_SYNC_DATA.rooms.join[roomId].ephemeral.events.push(e); + } + else { + NEXT_SYNC_DATA.rooms.join[roomId].timeline.events.push(e); + } + }); + } + beforeEach(function(done) { utils.beforeEach(this); httpBackend = new HttpBackend(); @@ -60,18 +108,20 @@ describe("MatrixClient room timelines", function() { userId: userId, accessToken: accessToken }); - eventData = { - chunk: [], - end: "end_", - start: "start_" - }; + setNextSyncData(); httpBackend.when("GET", "/pushrules").respond(200, {}); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, function() { - return eventData; + httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" }); + httpBackend.when("GET", "/sync").respond(200, SYNC_DATA); + httpBackend.when("GET", "/sync").respond(200, function() { + return NEXT_SYNC_DATA; }); client.startClient(); - httpBackend.flush("/pushrules").done(done); + httpBackend.flush("/pushrules").then(function() { + console.log("Flushed push"); + return httpBackend.flush("/filter"); + }).done(function () { + console.log("ALL READY"); done(); + }); }); afterEach(function() { @@ -97,11 +147,11 @@ describe("MatrixClient room timelines", function() { expect(member.userId).toEqual(userId); expect(member.name).toEqual(userName); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { done(); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should be updated correctly when the send request finishes " + @@ -110,12 +160,12 @@ describe("MatrixClient room timelines", function() { httpBackend.when("PUT", "/txn1").respond(200, { event_id: eventId }); - eventData.chunk = [ - utils.mkMessage({ - body: "I am a fish", user: userId, room: roomId - }) - ]; - eventData.chunk[0].event_id = eventId; + + var ev = utils.mkMessage({ + body: "I am a fish", user: userId, room: roomId + }) + ev.event_id = eventId; + setNextSyncData([ev]); client.on("sync", function(state) { if (state !== "PREPARED") { return; } @@ -123,14 +173,14 @@ describe("MatrixClient room timelines", function() { client.sendTextMessage(roomId, "I am a fish", "txn1").done( function() { expect(room.timeline[1].getId()).toEqual(eventId); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { expect(room.timeline[1].getId()).toEqual(eventId); done(); }); }); httpBackend.flush("/txn1", 1); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should be updated correctly when the send request finishes " + @@ -139,18 +189,18 @@ describe("MatrixClient room timelines", function() { httpBackend.when("PUT", "/txn1").respond(200, { event_id: eventId }); - eventData.chunk = [ - utils.mkMessage({ - body: "I am a fish", user: userId, room: roomId - }) - ]; - eventData.chunk[0].event_id = eventId; + + var ev = utils.mkMessage({ + body: "I am a fish", user: userId, room: roomId + }) + ev.event_id = eventId; + setNextSyncData([ev]); client.on("sync", function(state) { if (state !== "PREPARED") { return; } var room = client.getRoom(roomId); var promise = client.sendTextMessage(roomId, "I am a fish", "txn1"); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { // expect 3rd msg, it doesn't know this is the request is just did expect(room.timeline.length).toEqual(3); httpBackend.flush("/txn1", 1); @@ -162,7 +212,7 @@ describe("MatrixClient room timelines", function() { }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); }); @@ -195,9 +245,9 @@ describe("MatrixClient room timelines", function() { }); httpBackend.flush("/messages", 1); - httpBackend.flush("/events", 1); + httpBackend.flush("/sync", 1); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should set the right event.sender values", function(done) { @@ -238,9 +288,9 @@ describe("MatrixClient room timelines", function() { }); httpBackend.flush("/messages", 1); - httpBackend.flush("/events", 1); + httpBackend.flush("/sync", 1); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should add it them to the right place in the timeline", function(done) { @@ -267,9 +317,9 @@ describe("MatrixClient room timelines", function() { }); httpBackend.flush("/messages", 1); - httpBackend.flush("/events", 1); + httpBackend.flush("/sync", 1); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should use 'end' as the next pagination token", function(done) { @@ -289,21 +339,23 @@ describe("MatrixClient room timelines", function() { expect(room.oldState.paginationToken).toEqual(sbEndTok); }); - httpBackend.flush("/events", 1); + httpBackend.flush("/sync", 1); httpBackend.flush("/messages", 1).done(function() { done(); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); }); describe("new events", function() { it("should be added to the right place in the timeline", function(done) { - eventData.chunk = [ + var eventData = [ utils.mkMessage({user: userId, room: roomId}), utils.mkMessage({user: userId, room: roomId}) ]; + setNextSyncData(eventData); + client.on("sync", function(state) { if (state !== "PREPARED") { return; } var room = client.getRoom(roomId); @@ -312,37 +364,40 @@ describe("MatrixClient room timelines", function() { client.on("Room.timeline", function(event, rm, toStart) { expect(toStart).toBe(false); expect(rm).toEqual(room); - expect(event.event).toEqual(eventData.chunk[index]); + expect(event.event).toEqual(eventData[index]); index += 1; }); httpBackend.flush("/messages", 1); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { expect(index).toEqual(2); expect(room.timeline[room.timeline.length - 1].event).toEqual( - eventData.chunk[1] + eventData[1] ); expect(room.timeline[room.timeline.length - 2].event).toEqual( - eventData.chunk[0] + eventData[0] ); done(); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should set the right event.sender values", function(done) { - eventData.chunk = [ + var eventData = [ utils.mkMessage({user: userId, room: roomId}), utils.mkMembership({ user: userId, room: roomId, mship: "join", name: "New Name" }), utils.mkMessage({user: userId, room: roomId}) ]; + eventData[1].__prev_event = USER_MEMBERSHIP_EVENT; + setNextSyncData(eventData); + client.on("sync", function(state) { if (state !== "PREPARED") { return; } var room = client.getRoom(roomId); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { var preNameEvent = room.timeline[room.timeline.length - 3]; var postNameEvent = room.timeline[room.timeline.length - 1]; expect(preNameEvent.sender.name).toEqual(userName); @@ -350,17 +405,18 @@ describe("MatrixClient room timelines", function() { done(); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should set the right room.name", function(done) { - eventData.chunk = [ - utils.mkEvent({ - user: userId, room: roomId, type: "m.room.name", content: { - name: "Room 2" - } - }) - ]; + var secondRoomNameEvent = utils.mkEvent({ + user: userId, room: roomId, type: "m.room.name", content: { + name: "Room 2" + } + }); + secondRoomNameEvent.__prev_event = ROOM_NAME_EVENT; + setNextSyncData([secondRoomNameEvent]); + client.on("sync", function(state) { if (state !== "PREPARED") { return; } var room = client.getRoom(roomId); @@ -369,32 +425,32 @@ describe("MatrixClient room timelines", function() { nameEmitCount += 1; }); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { expect(nameEmitCount).toEqual(1); expect(room.name).toEqual("Room 2"); // do another round - eventData.chunk = [ - utils.mkEvent({ - user: userId, room: roomId, type: "m.room.name", content: { - name: "Room 3" - } - }) - ]; - httpBackend.when("GET", "/events").respond(200, eventData); - httpBackend.flush("/events", 1).done(function() { + var thirdRoomNameEvent = utils.mkEvent({ + user: userId, room: roomId, type: "m.room.name", content: { + name: "Room 3" + } + }); + thirdRoomNameEvent.__prev_event = secondRoomNameEvent; + setNextSyncData([thirdRoomNameEvent]); + httpBackend.when("GET", "/sync").respond(200, NEXT_SYNC_DATA); + httpBackend.flush("/sync", 1).done(function() { expect(nameEmitCount).toEqual(2); expect(room.name).toEqual("Room 3"); done(); }); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); it("should set the right room members", function(done) { var userC = "@cee:bar"; var userD = "@dee:bar"; - eventData.chunk = [ + var eventData = [ utils.mkMembership({ user: userC, room: roomId, mship: "join", name: "C" }), @@ -402,10 +458,14 @@ describe("MatrixClient room timelines", function() { user: userC, room: roomId, mship: "invite", skey: userD }) ]; + eventData[0].__prev_event = null; + eventData[1].__prev_event = null; + setNextSyncData(eventData); + client.on("sync", function(state) { if (state !== "PREPARED") { return; } var room = client.getRoom(roomId); - httpBackend.flush("/events", 1).done(function() { + httpBackend.flush("/sync", 1).done(function() { expect(room.currentState.getMembers().length).toEqual(4); expect(room.currentState.getMember(userC).name).toEqual("C"); expect(room.currentState.getMember(userC).membership).toEqual( @@ -418,7 +478,7 @@ describe("MatrixClient room timelines", function() { done(); }); }); - httpBackend.flush("/initialSync", 1); + httpBackend.flush("/sync", 1); }); }); }); From 3d9d31d6b13427582fbca03d13a934a40178953f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 13:22:27 +0000 Subject: [PATCH 17/32] Fix remaining integration tests --- lib/sync.js | 5 +- spec/integ/matrix-client-syncing.spec.js | 468 +++++++++++------------ spec/test-utils.js | 2 +- 3 files changed, 220 insertions(+), 255 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index 52371fc7c..ce08eccba 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -283,12 +283,13 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { var ephemeralEvents = self._mapSyncEventsFormat(joinObj.ephemeral); var accountDataEvents = self._mapSyncEventsFormat(joinObj.account_data); - if (joinObj.timeline.limited) { + if (joinObj.timeline && joinObj.timeline.limited) { // nuke the timeline so we don't get holes room.timeline = []; } self._processRoomEvents( - room, stateEvents, timelineEvents, joinObj.timeline.prev_batch + room, stateEvents, timelineEvents, + joinObj.timeline ? joinObj.timeline.prev_batch : null ); room.addEvents(ephemeralEvents); room.addEvents(accountDataEvents); diff --git a/spec/integ/matrix-client-syncing.spec.js b/spec/integ/matrix-client-syncing.spec.js index 7238f6429..ba222db2b 100644 --- a/spec/integ/matrix-client-syncing.spec.js +++ b/spec/integ/matrix-client-syncing.spec.js @@ -26,6 +26,7 @@ describe("MatrixClient syncing", function() { accessToken: selfAccessToken }); httpBackend.when("GET", "/pushrules").respond(200, {}); + httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" }); }); afterEach(function() { @@ -33,20 +34,14 @@ describe("MatrixClient syncing", function() { }); describe("startClient", function() { - var initialSync = { - end: "s_5_3", - presence: [], - rooms: [] - }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [] + var syncData = { + next_batch: "batch_token", + rooms: {}, + presence: {} }; - it("should start with /initialSync then move onto /events.", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + it("should /sync after /pushrules and /filter.", function(done) { + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient(); @@ -55,12 +50,12 @@ describe("MatrixClient syncing", function() { }); }); - it("should pass the 'end' token from /initialSync to the from= param " + - " of /events", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").check(function(req) { - expect(req.queryParams.from).toEqual(initialSync.end); - }).respond(200, eventData); + it("should pass the 'next_batch' token from /sync to the since= param " + + " of the next /sync", function(done) { + httpBackend.when("GET", "/sync").respond(200, syncData); + httpBackend.when("GET", "/sync").check(function(req) { + expect(req.queryParams.since).toEqual(syncData.next_batch); + }).respond(200, syncData); client.startClient(); @@ -71,56 +66,56 @@ describe("MatrixClient syncing", function() { }); describe("resolving invites to profile info", function() { - var initialSync = { - end: "s_5_3", - presence: [], - rooms: [{ - membership: "join", - room_id: roomOne, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ + + var syncData = { + next_batch: "s_5_3", + presence: { + events: [] + }, + rooms: { + join: { + + } + } + }; + + beforeEach(function() { + syncData.presence.events = []; + syncData.rooms.join[roomOne] = { + timeline: { + events: [ utils.mkMessage({ room: roomOne, user: otherUserId, msg: "hello" }) ] }, - state: [ - utils.mkMembership({ - room: roomOne, mship: "join", user: otherUserId - }), - utils.mkMembership({ - room: roomOne, mship: "join", user: selfUserId - }), - utils.mkEvent({ - type: "m.room.create", room: roomOne, user: selfUserId, - content: { - creator: selfUserId - } - }) - ] - }] - }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [] - }; - - beforeEach(function() { - eventData.chunk = []; + state: { + events: [ + utils.mkMembership({ + room: roomOne, mship: "join", user: otherUserId + }), + utils.mkMembership({ + room: roomOne, mship: "join", user: selfUserId + }), + utils.mkEvent({ + type: "m.room.create", room: roomOne, user: selfUserId, + content: { + creator: selfUserId + } + }) + ] + } + }; }); - it("should resolve incoming invites from /events", function(done) { - eventData.chunk = [ + it("should resolve incoming invites from /sync", function(done) { + syncData.rooms.join[roomOne].state.events.push( utils.mkMembership({ room: roomOne, mship: "invite", user: userC }) - ]; + ); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); httpBackend.when("GET", "/profile/" + encodeURIComponent(userC)).respond( 200, { avatar_url: "mxc://flibble/wibble", @@ -143,17 +138,18 @@ describe("MatrixClient syncing", function() { }); it("should use cached values from m.presence wherever possible", function(done) { - eventData.chunk = [ + syncData.presence.events = [ utils.mkPresence({ user: userC, presence: "online", name: "The Ghost" }), + ]; + syncData.rooms.join[roomOne].state.events.push( utils.mkMembership({ room: roomOne, mship: "invite", user: userC }) - ]; + ); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient({ resolveInvitesToProfiles: true @@ -167,17 +163,18 @@ describe("MatrixClient syncing", function() { }); it("should result in events on the room member firing", function(done) { - eventData.chunk = [ + syncData.presence.events = [ utils.mkPresence({ user: userC, presence: "online", name: "The Ghost" - }), + }) + ]; + syncData.rooms.join[roomOne].state.events.push( utils.mkMembership({ room: roomOne, mship: "invite", user: userC }) - ]; + ); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); var latestFiredName = null; client.on("RoomMember.name", function(event, m) { @@ -197,14 +194,13 @@ describe("MatrixClient syncing", function() { }); it("should no-op if resolveInvitesToProfiles is not set", function(done) { - eventData.chunk = [ + syncData.rooms.join[roomOne].state.events.push( utils.mkMembership({ room: roomOne, mship: "invite", user: userC }) - ]; + ); - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient(); @@ -220,44 +216,29 @@ describe("MatrixClient syncing", function() { }); describe("users", function() { - var initialSync = { - end: "s_5_3", - presence: [ - utils.mkPresence({ - user: userA, presence: "online" - }), - utils.mkPresence({ - user: userB, presence: "unavailable" - }) - ], - rooms: [] - }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [ - // existing user change - utils.mkPresence({ - user: userA, presence: "offline" - }), - // new user C - utils.mkPresence({ - user: userC, presence: "online" - }) - ] + var syncData = { + next_batch: "nb", + presence: { + events: [ + utils.mkPresence({ + user: userA, presence: "online" + }), + utils.mkPresence({ + user: userB, presence: "unavailable" + }) + ] + } }; - it("should create users for presence events from /initialSync and /events", + it("should create users for presence events from /sync", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient(); httpBackend.flush().done(function() { - expect(client.getUser(userA).presence).toEqual("offline"); + expect(client.getUser(userA).presence).toEqual("online"); expect(client.getUser(userB).presence).toEqual("unavailable"); - expect(client.getUser(userC).presence).toEqual("online"); done(); }); }); @@ -266,108 +247,128 @@ describe("MatrixClient syncing", function() { describe("room state", function() { var msgText = "some text here"; var otherDisplayName = "Bob Smith"; - var initialSync = { - end: "s_5_3", - presence: [], - rooms: [ - { - membership: "join", - room_id: roomOne, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ - utils.mkMessage({ - room: roomOne, user: otherUserId, msg: "hello" - }) - ] - }, - state: [ - utils.mkEvent({ - type: "m.room.name", room: roomOne, user: otherUserId, - content: { - name: "Old room name" - } - }), - utils.mkMembership({ - room: roomOne, mship: "join", user: otherUserId - }), - utils.mkMembership({ - room: roomOne, mship: "join", user: selfUserId - }), - utils.mkEvent({ - type: "m.room.create", room: roomOne, user: selfUserId, - content: { - creator: selfUserId - } - }) - ] - }, - { - membership: "join", - room_id: roomTwo, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ - utils.mkMessage({ - room: roomTwo, user: otherUserId, msg: "hiii" - }) - ] - }, - state: [ - utils.mkMembership({ - room: roomTwo, mship: "join", user: otherUserId, - name: otherDisplayName - }), - utils.mkMembership({ - room: roomTwo, mship: "join", user: selfUserId - }), - utils.mkEvent({ - type: "m.room.create", room: roomTwo, user: selfUserId, - content: { - creator: selfUserId - } - }) - ] + + var syncData = { + rooms: { + join: { + } - ] + } }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [ - utils.mkEvent({ - type: "m.room.name", room: roomOne, user: selfUserId, - content: { name: "A new room name" } - }), - utils.mkMessage({ - room: roomTwo, user: otherUserId, msg: msgText - }), - utils.mkEvent({ - type: "m.typing", room: roomTwo, - content: { user_ids: [otherUserId] } - }) - ] + syncData.rooms.join[roomOne] = { + timeline: { + events: [ + utils.mkMessage({ + room: roomOne, user: otherUserId, msg: "hello" + }) + ] + }, + state: { + events: [ + utils.mkEvent({ + type: "m.room.name", room: roomOne, user: otherUserId, + content: { + name: "Old room name" + } + }), + utils.mkMembership({ + room: roomOne, mship: "join", user: otherUserId + }), + utils.mkMembership({ + room: roomOne, mship: "join", user: selfUserId + }), + utils.mkEvent({ + type: "m.room.create", room: roomOne, user: selfUserId, + content: { + creator: selfUserId + } + }) + ] + } + }; + syncData.rooms.join[roomTwo] = { + timeline: { + events: [ + utils.mkMessage({ + room: roomTwo, user: otherUserId, msg: "hiii" + }) + ] + }, + state: { + events: [ + utils.mkMembership({ + room: roomTwo, mship: "join", user: otherUserId, + name: otherDisplayName + }), + utils.mkMembership({ + room: roomTwo, mship: "join", user: selfUserId + }), + utils.mkEvent({ + type: "m.room.create", room: roomTwo, user: selfUserId, + content: { + creator: selfUserId + } + }) + ] + } + }; + + var nextSyncData = { + rooms: { + join: { + + } + } + }; + + nextSyncData.rooms.join[roomOne] = { + state: { + events: [ + utils.mkEvent({ + type: "m.room.name", room: roomOne, user: selfUserId, + content: { name: "A new room name" } + }) + ] + } + }; + + nextSyncData.rooms.join[roomTwo] = { + timeline: { + events: [ + utils.mkMessage({ + room: roomTwo, user: otherUserId, msg: msgText + }) + ] + }, + ephemeral: { + events: [ + utils.mkEvent({ + type: "m.typing", room: roomTwo, + content: { user_ids: [otherUserId] } + }) + ] + } }; it("should continually recalculate the right room name.", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); + httpBackend.when("GET", "/sync").respond(200, nextSyncData); client.startClient(); httpBackend.flush().done(function() { var room = client.getRoom(roomOne); // should have clobbered the name to the one from /events - expect(room.name).toEqual(eventData.chunk[0].content.name); + expect(room.name).toEqual( + nextSyncData.rooms.join[roomOne].state.events[0].content.name + ); done(); }); }); it("should store the right events in the timeline.", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); + httpBackend.when("GET", "/sync").respond(200, nextSyncData); client.startClient(); @@ -381,8 +382,8 @@ describe("MatrixClient syncing", function() { }); it("should set the right room name.", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); + httpBackend.when("GET", "/sync").respond(200, nextSyncData); client.startClient(); httpBackend.flush().done(function() { @@ -394,8 +395,8 @@ describe("MatrixClient syncing", function() { }); it("should set the right user's typing flag.", function(done) { - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); + httpBackend.when("GET", "/sync").respond(200, nextSyncData); client.startClient(); @@ -421,23 +422,23 @@ describe("MatrixClient syncing", function() { }); describe("receipts", function() { - var initialSync = { - end: "s_5_3", - presence: [], - receipts: [], - rooms: [{ - membership: "join", - room_id: roomOne, - messages: { - start: "f_1_1", - end: "f_2_2", - chunk: [ - utils.mkMessage({ - room: roomOne, user: otherUserId, msg: "hello" - }) - ] - }, - state: [ + var syncData = { + rooms: { + join: { + + } + } + }; + syncData.rooms.join[roomOne] = { + timeline: { + events: [ + utils.mkMessage({ + room: roomOne, user: otherUserId, msg: "hello" + }) + ] + }, + state: { + events: [ utils.mkEvent({ type: "m.room.name", room: roomOne, user: otherUserId, content: { @@ -457,21 +458,17 @@ describe("MatrixClient syncing", function() { } }) ] - }] - }; - var eventData = { - start: "s_5_3", - end: "e_6_7", - chunk: [] + } }; beforeEach(function() { - eventData.chunk = []; - initialSync.receipts = []; + syncData.rooms.join[roomOne].ephemeral = { + events: [] + }; }); - it("should sync receipts from /initialSync.", function(done) { - var ackEvent = initialSync.rooms[0].messages.chunk[0]; + it("should sync receipts from /sync.", function(done) { + var ackEvent = syncData.rooms.join[roomOne].timeline.events[0]; var receipt = {}; receipt[ackEvent.event_id] = { "m.read": {} @@ -479,45 +476,12 @@ describe("MatrixClient syncing", function() { receipt[ackEvent.event_id]["m.read"][otherUserId] = { ts: 176592842636 }; - initialSync.receipts = [{ + syncData.rooms.join[roomOne].ephemeral.events = [{ content: receipt, room_id: roomOne, type: "m.receipt" }]; - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); - - client.startClient(); - - httpBackend.flush().done(function() { - var room = client.getRoom(roomOne); - expect(room.getReceiptsForEvent(new MatrixEvent(ackEvent))).toEqual([{ - type: "m.read", - userId: otherUserId, - data: { - ts: 176592842636 - } - }]); - done(); - }); - }); - - it("should sync receipts from /events.", function(done) { - var ackEvent = initialSync.rooms[0].messages.chunk[0]; - var receipt = {}; - receipt[ackEvent.event_id] = { - "m.read": {} - }; - receipt[ackEvent.event_id]["m.read"][otherUserId] = { - ts: 176592842636 - }; - eventData.chunk = [{ - content: receipt, - room_id: roomOne, - type: "m.receipt" - }]; - httpBackend.when("GET", "/initialSync").respond(200, initialSync); - httpBackend.when("GET", "/events").respond(200, eventData); + httpBackend.when("GET", "/sync").respond(200, syncData); client.startClient(); diff --git a/spec/test-utils.js b/spec/test-utils.js index 92a74107f..d8c6fbce3 100644 --- a/spec/test-utils.js +++ b/spec/test-utils.js @@ -88,8 +88,8 @@ module.exports.mkPresence = function(opts) { var event = { event_id: "$" + Math.random() + "-" + Math.random(), type: "m.presence", + sender: opts.user, content: { - user_id: opts.user, avatar_url: opts.url, displayname: opts.name, last_active_ago: opts.ago, From de32ac0c441da3ea5016a3aa0fa23c3392bb8979 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 13:23:46 +0000 Subject: [PATCH 18/32] Fix linting --- spec/integ/matrix-client-event-emitter.spec.js | 2 -- spec/integ/matrix-client-room-timeline.spec.js | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/spec/integ/matrix-client-event-emitter.spec.js b/spec/integ/matrix-client-event-emitter.spec.js index e249ce149..eee5be4ea 100644 --- a/spec/integ/matrix-client-event-emitter.spec.js +++ b/spec/integ/matrix-client-event-emitter.spec.js @@ -107,8 +107,6 @@ describe("MatrixClient events", function() { NEXT_SYNC_DATA.rooms.join["!erufh:bar"].ephemeral.events ); - - var chunkIndex = 0; client.on("event", function(event) { var found = false; for (var i = 0; i < expectedEvents.length; i++) { diff --git a/spec/integ/matrix-client-room-timeline.spec.js b/spec/integ/matrix-client-room-timeline.spec.js index 59a45704c..0ad7ce7a7 100644 --- a/spec/integ/matrix-client-room-timeline.spec.js +++ b/spec/integ/matrix-client-room-timeline.spec.js @@ -163,7 +163,7 @@ describe("MatrixClient room timelines", function() { var ev = utils.mkMessage({ body: "I am a fish", user: userId, room: roomId - }) + }); ev.event_id = eventId; setNextSyncData([ev]); @@ -192,7 +192,7 @@ describe("MatrixClient room timelines", function() { var ev = utils.mkMessage({ body: "I am a fish", user: userId, room: roomId - }) + }); ev.event_id = eventId; setNextSyncData([ev]); From 7c43d15ea584a947c3145cd6e21db8ec95943226 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 13:31:26 +0000 Subject: [PATCH 19/32] More linting; crypto test fix --- spec/integ/matrix-client-crypto.spec.js | 32 +++++++++++-------- .../integ/matrix-client-event-emitter.spec.js | 7 ++-- .../integ/matrix-client-room-timeline.spec.js | 8 ++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/spec/integ/matrix-client-crypto.spec.js b/spec/integ/matrix-client-crypto.spec.js index 75b559c36..9717de1f7 100644 --- a/spec/integ/matrix-client-crypto.spec.js +++ b/spec/integ/matrix-client-crypto.spec.js @@ -200,22 +200,26 @@ describe("MatrixClient crypto", function() { }); function bobRecvMessage(done) { - var initialSync = { - end: "alpha", - presence: [], - rooms: [] + var syncData = { + next_batch: "x", + rooms: { + join: { + + } + } }; - var events = { - start: "alpha", - end: "beta", - chunk: [utils.mkEvent({ - type: "m.room.encrypted", - room: roomId, - content: aliMessage - })] + syncData.rooms.join[roomId] = { + timeline: { + events: [ + utils.mkEvent({ + type: "m.room.encrypted", + room: roomId, + content: aliMessage + }) + ] + } }; - httpBackend.when("GET", "initialSync").respond(200, initialSync); - httpBackend.when("GET", "events").respond(200, events); + httpBackend.when("GET", "/sync").respond(200, syncData); bobClient.on("event", function(event) { expect(event.getType()).toEqual("m.room.message"); expect(event.getContent()).toEqual({ diff --git a/spec/integ/matrix-client-event-emitter.spec.js b/spec/integ/matrix-client-event-emitter.spec.js index eee5be4ea..e5191d97c 100644 --- a/spec/integ/matrix-client-event-emitter.spec.js +++ b/spec/integ/matrix-client-event-emitter.spec.js @@ -53,7 +53,8 @@ describe("MatrixClient events", function() { room: "!erufh:bar", mship: "join", user: "@foo:bar" }), utils.mkEvent({ - type: "m.room.create", room: "!erufh:bar", user: "@foo:bar", + type: "m.room.create", room: "!erufh:bar", + user: "@foo:bar", content: { creator: "@foo:bar" } @@ -116,7 +117,9 @@ describe("MatrixClient events", function() { break; } } - expect(found).toBe(true, "Unexpected 'event' emitted: " + event.getType()); + expect(found).toBe( + true, "Unexpected 'event' emitted: " + event.getType() + ); }); client.startClient(); diff --git a/spec/integ/matrix-client-room-timeline.spec.js b/spec/integ/matrix-client-room-timeline.spec.js index 0ad7ce7a7..a8cf97f70 100644 --- a/spec/integ/matrix-client-room-timeline.spec.js +++ b/spec/integ/matrix-client-room-timeline.spec.js @@ -39,7 +39,8 @@ describe("MatrixClient room timelines", function() { events: [ ROOM_NAME_EVENT, utils.mkMembership({ - room: roomId, mship: "join", user: otherUserId, name: "Bob" + room: roomId, mship: "join", + user: otherUserId, name: "Bob" }), USER_MEMBERSHIP_EVENT, utils.mkEvent({ @@ -117,11 +118,8 @@ describe("MatrixClient room timelines", function() { }); client.startClient(); httpBackend.flush("/pushrules").then(function() { - console.log("Flushed push"); return httpBackend.flush("/filter"); - }).done(function () { - console.log("ALL READY"); done(); - }); + }).done(done); }); afterEach(function() { From eedff29acb9d18eba77db30949d80b2cf3e85ee2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 13:35:46 +0000 Subject: [PATCH 20/32] Add filter stub to crypto test --- spec/integ/matrix-client-crypto.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/integ/matrix-client-crypto.spec.js b/spec/integ/matrix-client-crypto.spec.js index 9717de1f7..d40658b5a 100644 --- a/spec/integ/matrix-client-crypto.spec.js +++ b/spec/integ/matrix-client-crypto.spec.js @@ -67,6 +67,7 @@ describe("MatrixClient crypto", function() { }); httpBackend.when("GET", "/pushrules").respond(200, {}); + httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" }); }); describe("Ali account setup", function() { From 465635444f80bd6216ee123e861a65f81645aacf Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 15:07:40 +0000 Subject: [PATCH 21/32] s/user_id/sender/g in tests --- spec/test-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/test-utils.js b/spec/test-utils.js index d8c6fbce3..4f4e26573 100644 --- a/spec/test-utils.js +++ b/spec/test-utils.js @@ -61,7 +61,7 @@ module.exports.mkEvent = function(opts) { var event = { type: opts.type, room_id: opts.room, - user_id: opts.user, + sender: opts.user, content: opts.content, event_id: "$" + Math.random() + "-" + Math.random() }; From a6f64b5f031467cb22023134053503948db9fa00 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 11 Dec 2015 15:27:40 +0000 Subject: [PATCH 22/32] v2 filter test --- spec/unit/matrix-client.spec.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/spec/unit/matrix-client.spec.js b/spec/unit/matrix-client.spec.js index 16d3330b4..daa67ad9d 100644 --- a/spec/unit/matrix-client.spec.js +++ b/spec/unit/matrix-client.spec.js @@ -159,6 +159,23 @@ describe("MatrixClient", function() { }); }); + it("should not POST /filter if a filter already exists", function(done) { + httpLookups = []; + httpLookups.push(PUSH_RULES_RESPONSE); + httpLookups.push(SYNC_RESPONSE); + var filterId = "ehfewf"; + store.getFilterIdByName.andReturn(filterId); + client.startClient(); + + client.on("sync", function syncListener(state) { + if (state === "SYNCING") { + expect(httpLookups.length).toEqual(0); + client.removeListener("sync", syncListener); + done(); + } + }); + }); + describe("getSyncState", function() { it("should return null if the client isn't started", function() { From d6299b634c649ae0d37c7960b45ea797f7502893 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 09:22:02 +0000 Subject: [PATCH 23/32] Scope filter keys in localStorage on user_id --- lib/sync.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index ce08eccba..854212356 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -15,7 +15,11 @@ var utils = require("./utils"); var httpApi = require("./http-api"); var Filter = require("./filter"); -var FILTER_SYNC = "FILTER_SYNC"; +function getFilterName(userId) { + // scope this on the user ID because people may login on many accounts + // and they all need to be stored! + return "FILTER_SYNC_" + userId; +} /** * Internal class - unstable. @@ -96,7 +100,7 @@ SyncApi.prototype.sync = function() { // Get or create filter - var filterId = client.store.getFilterIdByName(FILTER_SYNC); + var filterId = client.store.getFilterIdByName(getFilterName(client.credentials.userId)); if (filterId) { // super, just use that. console.log("Using existing filter ID %s", filterId); @@ -108,7 +112,9 @@ SyncApi.prototype.sync = function() { var filter = new Filter(client.credentials.userId); filter.setTimelineLimit(self.opts.initialSyncLimit); client.createFilter(filter.getDefinition()).done(function(filter) { - client.store.setFilterIdByName(FILTER_SYNC, filter.filterId); + client.store.setFilterIdByName( + getFilterName(client.credentials.userId), filter.filterId + ); console.log("Created filter ", filter.filterId); self._sync({ filterId: filter.filterId }); // Now start the /sync loop }, retryHandler(attempt, getFilter)); From 9c49d26525885f0e29c94fd698230ca938dca945 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 09:24:33 +0000 Subject: [PATCH 24/32] Linting --- lib/sync.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/sync.js b/lib/sync.js index 854212356..7db95cd1e 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -100,7 +100,9 @@ SyncApi.prototype.sync = function() { // Get or create filter - var filterId = client.store.getFilterIdByName(getFilterName(client.credentials.userId)); + var filterId = client.store.getFilterIdByName( + getFilterName(client.credentials.userId) + ); if (filterId) { // super, just use that. console.log("Using existing filter ID %s", filterId); From fc6ce20e141a6bf353aa33874a27dcd2fadc4d52 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 10:27:53 +0000 Subject: [PATCH 25/32] Check unsigned.age for getAge() for v2. Don't spam SYNCING emissions. --- lib/models/event.js | 5 ++++- lib/sync.js | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/models/event.js b/lib/models/event.js index dd0d4623e..3c7dba3e6 100644 --- a/lib/models/event.js +++ b/lib/models/event.js @@ -143,7 +143,10 @@ module.exports.MatrixEvent.prototype = { * @return {Number} The age of this event in milliseconds. */ getAge: function() { - return this.event.age; + if (this.event.unsigned && this.event.unsigned.age) { + return this.event.unsigned.age; // v2 + } + return this.event.age; // v1 }, /** diff --git a/lib/sync.js b/lib/sync.js index 7db95cd1e..14341c649 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -15,6 +15,12 @@ var utils = require("./utils"); var httpApi = require("./http-api"); var Filter = require("./filter"); +// /sync requests allow you to set a timeout= but the request may continue +// beyond that and wedge forever, so we need to track how long we are willing +// to keep open the connection. This constant is *ADDED* to the timeout= value +// to determine the max time we're willing to wait. +var BUFFER_PERIOD_MS = 90 * 1000; + function getFilterName(userId) { // scope this on the user ID because people may login on many accounts // and they all need to be stored! @@ -185,7 +191,7 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { var timeoutObj = setTimeout(function() { discardResult = true; errHandler("Locally timed out waiting for a response"); - }, qps.timeout + (20 * 1000)); // 20s buffer + }, qps.timeout + BUFFER_PERIOD_MS); client._http.authedRequestWithPrefix( undefined, "GET", "/sync", qps, undefined, httpApi.PREFIX_V2_ALPHA @@ -324,7 +330,9 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { updateSyncState(client, "PREPARED"); syncOptions.hasSyncedBefore = true; } - updateSyncState(client, "SYNCING"); + if (client._syncState !== "SYNCING") { + updateSyncState(client, "SYNCING"); + } self._sync(syncOptions); }, errHandler); }; From 13cab79e04d5c08a4eee625f0211eadf60d83eb9 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 10:31:27 +0000 Subject: [PATCH 26/32] Revert prev commit - emit SYNCING>SYNCING and now comment why we do this --- lib/sync.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index 14341c649..cd4c6e22a 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -330,9 +330,10 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { updateSyncState(client, "PREPARED"); syncOptions.hasSyncedBefore = true; } - if (client._syncState !== "SYNCING") { - updateSyncState(client, "SYNCING"); - } + + // keep emitting SYNCING -> SYNCING for clients who want to do bulk updates + updateSyncState(client, "SYNCING"); + self._sync(syncOptions); }, errHandler); }; From fcf1abb18578673e3586184f6d3409c5d1d3006f Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 11:35:50 +0000 Subject: [PATCH 27/32] Use v2 transaction IDs to suppress dupes without linear scans of the timeline! --- lib/models/event.js | 9 +++++---- lib/models/room.js | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/models/event.js b/lib/models/event.js index 3c7dba3e6..e8db3dbf2 100644 --- a/lib/models/event.js +++ b/lib/models/event.js @@ -143,10 +143,7 @@ module.exports.MatrixEvent.prototype = { * @return {Number} The age of this event in milliseconds. */ getAge: function() { - if (this.event.unsigned && this.event.unsigned.age) { - return this.event.unsigned.age; // v2 - } - return this.event.age; // v1 + return this.getUnsigned().age || this.event.age; // v2 / v1 }, /** @@ -172,5 +169,9 @@ module.exports.MatrixEvent.prototype = { */ isEncrypted: function() { return this.encrypted; + }, + + getUnsigned: function() { + return this.event.unsigned || {}; } }; diff --git a/lib/models/room.js b/lib/models/room.js index a2f8a1e0f..b04d78d23 100644 --- a/lib/models/room.js +++ b/lib/models/room.js @@ -79,6 +79,7 @@ function Room(roomId, opts) { this.storageToken = opts.storageToken; this._opts = opts; this._redactions = []; + this._txnToEvent = {}; // Pending in-flight requests { string: MatrixEvent } // receipts should clobber based on receipt_type and user_id pairs hence // the form of this structure. This is sub-optimal for the exposed APIs // which pass in an event ID and get back some receipts, so we also store @@ -211,6 +212,23 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline) { ) ); + // FIXME: HORRIBLE ASSUMPTION THAT THIS PROP EXISTS + // Exists due to client.js:815 (MatrixClient.sendEvent) + // We should make txnId a first class citizen. + if (!toStartOfTimeline && events[i]._txnId) { + this._txnToEvent[events[i]._txnId] = events[i]; + } + else if(!toStartOfTimeline && events[i].getUnsigned().transaction_id) { + var existingEvent = this._txnToEvent[events[i].getUnsigned().transaction_id]; + if (existingEvent) { + // no longer pending + delete this._txnToEvent[events[i].getUnsigned().transaction_id]; + // replace the event source + existingEvent.event = events[i].event; + continue; + } + } + setEventMetadata(events[i], stateContext, toStartOfTimeline); // modify state if (events[i].isState()) { From a00e318d735b0772f208bc061517686f81a2ab2b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 12:28:04 +0000 Subject: [PATCH 28/32] Fix pagination - set prev_batch at the right time --- lib/sync.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index cd4c6e22a..9335cdd8f 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -301,9 +301,16 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { // nuke the timeline so we don't get holes room.timeline = []; } + + // we want to set a new pagination token if this is the first time + // we've made this room or if we're nuking the timeline + var paginationToken = null; + if (joinObj.isBrandNewRoom || joinObj.timeline.limited) { + paginationToken = joinObj.timeline.prev_batch; + } + self._processRoomEvents( - room, stateEvents, timelineEvents, - joinObj.timeline ? joinObj.timeline.prev_batch : null + room, stateEvents, timelineEvents, paginationToken ); room.addEvents(ephemeralEvents); room.addEvents(accountDataEvents); From bd4d8433abe8cae32f1b5b4ab6e41970de307f9b Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 14:11:25 +0000 Subject: [PATCH 29/32] Fix catchup bugs caused by using a stale pagination token --- lib/sync.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/sync.js b/lib/sync.js index 9335cdd8f..824b8a15a 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -456,6 +456,14 @@ SyncApi.prototype._processRoomEvents = function(room, stateEventList, ); var stateEvents = stateEventList; + // Set the pagination token BEFORE adding events to the timeline: it's not + // unreasonable for clients to call scrollback() in response to Room.timeline + // events which addEventsToTimeline will emit-- we want to make sure they use + // the right token if and when they do. + if (paginationToken) { + room.oldState.paginationToken = paginationToken; + } + // set the state of the room to as it was before the timeline executes room.oldState.setStateEvents(oldStateEvents); room.currentState.setStateEvents(stateEvents); @@ -465,9 +473,7 @@ SyncApi.prototype._processRoomEvents = function(room, stateEventList, // execute the timeline events, this will begin to diverge the current state // if the timeline has any state events in it. room.addEventsToTimeline(timelineEventList); - if (paginationToken) { - room.oldState.paginationToken = paginationToken; - } + }; function retryTimeMsForAttempt(attempt) { From f48c9175e56f124fa9319d8aacb90de84b7247be Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 14:12:49 +0000 Subject: [PATCH 30/32] Linting --- lib/models/room.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/models/room.js b/lib/models/room.js index b04d78d23..7bb9d6a33 100644 --- a/lib/models/room.js +++ b/lib/models/room.js @@ -218,7 +218,7 @@ Room.prototype.addEventsToTimeline = function(events, toStartOfTimeline) { if (!toStartOfTimeline && events[i]._txnId) { this._txnToEvent[events[i]._txnId] = events[i]; } - else if(!toStartOfTimeline && events[i].getUnsigned().transaction_id) { + else if (!toStartOfTimeline && events[i].getUnsigned().transaction_id) { var existingEvent = this._txnToEvent[events[i].getUnsigned().transaction_id]; if (existingEvent) { // no longer pending From 46504b8b9f899a46e2fc62c94125b48054c2a3b4 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Mon, 14 Dec 2015 14:16:47 +0000 Subject: [PATCH 31/32] Fix tests; need more paranoia --- lib/sync.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/sync.js b/lib/sync.js index 824b8a15a..6a20a070c 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -297,7 +297,9 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { var ephemeralEvents = self._mapSyncEventsFormat(joinObj.ephemeral); var accountDataEvents = self._mapSyncEventsFormat(joinObj.account_data); - if (joinObj.timeline && joinObj.timeline.limited) { + joinObj.timeline = joinObj.timeline || {}; + + if (joinObj.timeline.limited) { // nuke the timeline so we don't get holes room.timeline = []; } From bfda04daea0bdcddc87a4d58627d560406c1a5e2 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Tue, 15 Dec 2015 11:59:41 +0000 Subject: [PATCH 32/32] Move local timeout logic to the HTTP API class. Fixes /sync bug The ability to set a local timeout is applicable to any request, so move it to http-api.js - We only use this on /sync requests currently. This simplifies the SyncApi since it doesn't need to worry about it anymore -- the request promise just gets rejected if the timer expires. Whilst testing I noticed a weird anomaly which I cannot explain. Playing with Chrome's network debugger, once you recover from a black hole (0kbps, 90s RTT) the subsequent requests take 20s to return *even though there is no throttling*. This was causing issues when using a local timer of timeout= and BUFFER_PERIOD_MS when timeout=1 due to attempts>1 - they were being knifed before the response could return. The 20s latency was entirely artifical (checked synapse logs and they were sub 1s), but I cannot find anywhere in the JS-SDK or browser-requests where this would be the cause. This persisted even when BUFFER_PERIOD_MS was changed from 20s to 10s. --- lib/http-api.js | 57 ++++++++++++++++++++++++++++++++++++++++--------- lib/sync.js | 36 +++++++++++-------------------- 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/lib/http-api.js b/lib/http-api.js index 6e487ef4a..1f4d42a43 100644 --- a/lib/http-api.js +++ b/lib/http-api.js @@ -242,6 +242,8 @@ module.exports.MatrixHttpApi.prototype = { * @param {Object} queryParams A dict of query params (these will NOT be * urlencoded). * @param {Object} data The HTTP JSON body. + * @param {Number=} localTimeoutMs The maximum amount of time to wait before + * timing out the request. If not specified, there is no timeout. * @return {module:client.Promise} Resolves to {data: {Object}, * headers: {Object}, code: {Number}}. * If onlyData is set, this will resolve to the data @@ -249,10 +251,10 @@ module.exports.MatrixHttpApi.prototype = { * @return {module:http-api.MatrixError} Rejects with an error if a problem * occurred. This includes network problems and Matrix-specific error JSON. */ - authedRequest: function(callback, method, path, queryParams, data) { + authedRequest: function(callback, method, path, queryParams, data, localTimeoutMs) { if (!queryParams) { queryParams = {}; } queryParams.access_token = this.opts.accessToken; - return this.request(callback, method, path, queryParams, data); + return this.request(callback, method, path, queryParams, data, localTimeoutMs); }, /** @@ -265,6 +267,8 @@ module.exports.MatrixHttpApi.prototype = { * @param {Object} queryParams A dict of query params (these will NOT be * urlencoded). * @param {Object} data The HTTP JSON body. + * @param {Number=} localTimeoutMs The maximum amount of time to wait before + * timing out the request. If not specified, there is no timeout. * @return {module:client.Promise} Resolves to {data: {Object}, * headers: {Object}, code: {Number}}. * If onlyData is set, this will resolve to the data @@ -272,9 +276,9 @@ module.exports.MatrixHttpApi.prototype = { * @return {module:http-api.MatrixError} Rejects with an error if a problem * occurred. This includes network problems and Matrix-specific error JSON. */ - request: function(callback, method, path, queryParams, data) { + request: function(callback, method, path, queryParams, data, localTimeoutMs) { return this.requestWithPrefix( - callback, method, path, queryParams, data, this.opts.prefix + callback, method, path, queryParams, data, this.opts.prefix, localTimeoutMs ); }, @@ -292,6 +296,8 @@ module.exports.MatrixHttpApi.prototype = { * @param {Object} data The HTTP JSON body. * @param {string} prefix The full prefix to use e.g. * "/_matrix/client/v2_alpha". + * @param {Number=} localTimeoutMs The maximum amount of time to wait before + * timing out the request. If not specified, there is no timeout. * @return {module:client.Promise} Resolves to {data: {Object}, * headers: {Object}, code: {Number}}. * If onlyData is set, this will resolve to the data @@ -300,13 +306,15 @@ module.exports.MatrixHttpApi.prototype = { * occurred. This includes network problems and Matrix-specific error JSON. */ authedRequestWithPrefix: function(callback, method, path, queryParams, data, - prefix) { + prefix, localTimeoutMs) { var fullUri = this.opts.baseUrl + prefix + path; if (!queryParams) { queryParams = {}; } queryParams.access_token = this.opts.accessToken; - return this._request(callback, method, fullUri, queryParams, data); + return this._request( + callback, method, fullUri, queryParams, data, localTimeoutMs + ); }, /** @@ -323,6 +331,8 @@ module.exports.MatrixHttpApi.prototype = { * @param {Object} data The HTTP JSON body. * @param {string} prefix The full prefix to use e.g. * "/_matrix/client/v2_alpha". + * @param {Number=} localTimeoutMs The maximum amount of time to wait before + * timing out the request. If not specified, there is no timeout. * @return {module:client.Promise} Resolves to {data: {Object}, * headers: {Object}, code: {Number}}. * If onlyData is set, this will resolve to the data @@ -330,12 +340,15 @@ module.exports.MatrixHttpApi.prototype = { * @return {module:http-api.MatrixError} Rejects with an error if a problem * occurred. This includes network problems and Matrix-specific error JSON. */ - requestWithPrefix: function(callback, method, path, queryParams, data, prefix) { + requestWithPrefix: function(callback, method, path, queryParams, data, prefix, + localTimeoutMs) { var fullUri = this.opts.baseUrl + prefix + path; if (!queryParams) { queryParams = {}; } - return this._request(callback, method, fullUri, queryParams, data); + return this._request( + callback, method, fullUri, queryParams, data, localTimeoutMs + ); }, /** @@ -357,12 +370,13 @@ module.exports.MatrixHttpApi.prototype = { return this.opts.baseUrl + prefix + path + queryString; }, - _request: function(callback, method, uri, queryParams, data) { + _request: function(callback, method, uri, queryParams, data, localTimeoutMs) { if (callback !== undefined && !utils.isFunction(callback)) { throw Error( "Expected callback to be a function but got " + typeof callback ); } + var self = this; if (!queryParams) { queryParams = {}; } @@ -373,6 +387,20 @@ module.exports.MatrixHttpApi.prototype = { } } var defer = q.defer(); + + var timeoutId; + var timedOut = false; + if (localTimeoutMs) { + timeoutId = setTimeout(function() { + timedOut = true; + defer.reject(new module.exports.MatrixError({ + error: "Locally timed out waiting for a response", + errcode: "ORG_MATRIX_JSSDK_TIMEOUT", + timeout: localTimeoutMs + })); + }, localTimeoutMs); + } + try { this.opts.request( { @@ -384,7 +412,16 @@ module.exports.MatrixHttpApi.prototype = { json: true, _matrix_opts: this.opts }, - requestCallback(defer, callback, this.opts.onlyData) + function(err, response, body) { + if (localTimeoutMs) { + clearTimeout(timeoutId); + if (timedOut) { + return; // already rejected promise + } + } + var handlerFn = requestCallback(defer, callback, self.opts.onlyData); + handlerFn(err, response, body); + } ); } catch (ex) { diff --git a/lib/sync.js b/lib/sync.js index 6a20a070c..c884bc6d4 100644 --- a/lib/sync.js +++ b/lib/sync.js @@ -19,7 +19,7 @@ var Filter = require("./filter"); // beyond that and wedge forever, so we need to track how long we are willing // to keep open the connection. This constant is *ADDED* to the timeout= value // to determine the max time we're willing to wait. -var BUFFER_PERIOD_MS = 90 * 1000; +var BUFFER_PERIOD_MS = 20 * 1000; function getFilterName(userId) { // scope this on the user ID because people may login on many accounts @@ -178,30 +178,10 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { qps.room_id = client._guestRooms; } - // Set up local timer and error handler for retries. - function errHandler(err) { - console.error("/sync error (%s attempts): %s", attempt, err); - attempt += 1; - startSyncingRetryTimer(client, attempt, function() { - self._sync(syncOptions, attempt); - }); - updateSyncState(client, "ERROR", { error: err }); - } - var discardResult = false; - var timeoutObj = setTimeout(function() { - discardResult = true; - errHandler("Locally timed out waiting for a response"); - }, qps.timeout + BUFFER_PERIOD_MS); - client._http.authedRequestWithPrefix( - undefined, "GET", "/sync", qps, undefined, httpApi.PREFIX_V2_ALPHA + undefined, "GET", "/sync", qps, undefined, httpApi.PREFIX_V2_ALPHA, + this.opts.pollTimeout + BUFFER_PERIOD_MS // normal timeout= plus buffer time ).done(function(data) { - if (discardResult) { - return; - } - else { - clearTimeout(timeoutObj); - } // data looks like: // { // next_batch: $token, @@ -344,7 +324,15 @@ SyncApi.prototype._sync = function(syncOptions, attempt) { updateSyncState(client, "SYNCING"); self._sync(syncOptions); - }, errHandler); + }, function(err) { + console.error("/sync error (%s attempts): %s", attempt, err); + console.error(err); + attempt += 1; + startSyncingRetryTimer(client, attempt, function() { + self._sync(syncOptions, attempt); + }); + updateSyncState(client, "ERROR", { error: err }); + }); }; /**