From 1ce9e7c6bb6a1a60f74097bd1ee43563121d3014 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Jan 2017 10:44:20 +0000 Subject: [PATCH 1/4] BREAKING CHANGE: Remove WebStorageStore This will be replaced with an IndexedDB style solution. Maintaining 2 different persistent stores is not my idea of fun. --- src/matrix.js | 3 - src/store/memory.js | 3 +- src/store/webstorage.js | 686 ---------------------------------------- 3 files changed, 1 insertion(+), 691 deletions(-) delete mode 100644 src/store/webstorage.js diff --git a/src/matrix.js b/src/matrix.js index 260f1d23f..3a7edcff4 100644 --- a/src/matrix.js +++ b/src/matrix.js @@ -21,9 +21,6 @@ module.exports.MatrixEvent = require("./models/event").MatrixEvent; module.exports.EventStatus = require("./models/event").EventStatus; /** The {@link module:store/memory.MatrixInMemoryStore|MatrixInMemoryStore} class. */ module.exports.MatrixInMemoryStore = require("./store/memory").MatrixInMemoryStore; -/** The {@link module:store/webstorage~WebStorageStore|WebStorageStore} class. - * Work in progress; unstable. */ -module.exports.WebStorageStore = require("./store/webstorage"); /** The {@link module:http-api.MatrixHttpApi|MatrixHttpApi} class. */ module.exports.MatrixHttpApi = require("./http-api").MatrixHttpApi; /** The {@link module:http-api.MatrixError|MatrixError} class. */ diff --git a/src/store/memory.js b/src/store/memory.js index 26bcc21c1..3c0386504 100644 --- a/src/store/memory.js +++ b/src/store/memory.js @@ -26,8 +26,7 @@ limitations under the License. * @constructor * @param {Object=} opts Config options * @param {LocalStorage} opts.localStorage The local storage instance to persist - * some forms of data such as tokens. Rooms will NOT be stored. See - * {@link WebStorageStore} to persist rooms. + * some forms of data such as tokens. Rooms will NOT be stored. */ module.exports.MatrixInMemoryStore = function MatrixInMemoryStore(opts) { opts = opts || {}; diff --git a/src/store/webstorage.js b/src/store/webstorage.js deleted file mode 100644 index 76d83924e..000000000 --- a/src/store/webstorage.js +++ /dev/null @@ -1,686 +0,0 @@ -/* -Copyright 2015, 2016 OpenMarket Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -"use strict"; -/** - * This is an internal module. Implementation details: - * 
- * Room data is stored as follows:
- *   room_$ROOMID_timeline_$INDEX : [ Event, Event, Event ]
- *   room_$ROOMID_state : {
- *                          pagination_token: ,
- *                          events: {
- *                            : {  : {JSON} }
- *                          }
- *                        }
- * User data is stored as follows:
- *   user_$USERID : User
- * Sync token:
- *   sync_token : $TOKEN
- *
- * Room Retrieval
- * --------------
- * Retrieving a room requires the $ROOMID which then pulls out the current state
- * from room_$ROOMID_state. A defined starting batch of timeline events are then
- * extracted from the highest numbered $INDEX for room_$ROOMID_timeline_$INDEX
- * (more indices as required). The $INDEX may be negative. These are
- * added to the timeline in the same way as /initialSync (old state will diverge).
- * If there exists a room_$ROOMID_timeline_live key, then a timeline sync should
- * be performed before retrieving.
- *
- * Retrieval of earlier messages
- * -----------------------------
- * The earliest event the Room instance knows about is E. Retrieving earlier
- * messages requires a Room which has a storageToken defined.
- * This token maps to the index I where the Room is at. Events are then retrieved from
- * room_$ROOMID_timeline_{I} and elements before E are extracted. If the limit
- * demands more events, I-1 is retrieved, up until I=min $INDEX where it gives
- * less than the limit. Index may go negative if you have paginated in the past.
- *
- * Full Insertion
- * --------------
- * Storing a room requires the timeline and state keys for $ROOMID to
- * be blown away and completely replaced, which is computationally expensive.
- * Room.timeline is batched according to the given batch size B. These batches
- * are then inserted into storage as room_$ROOMID_timeline_$INDEX. Finally,
- * the current room state is persisted to room_$ROOMID_state.
- *
- * Incremental Insertion
- * ---------------------
- * As events arrive, the store can quickly persist these new events. This
- * involves pushing the events to room_$ROOMID_timeline_live. If the
- * current room state has been modified by the new event, then
- * room_$ROOMID_state should be updated in addition to the timeline.
- *
- * Timeline sync
- * -------------
- * Retrieval of events from the timeline depends on the proper batching of
- * events. This is computationally expensive to perform on every new event, so
- * is deferred by inserting live events to room_$ROOMID_timeline_live. A
- * timeline sync reconciles timeline_live and timeline_$INDEX. This involves
- * retrieving _live and the highest numbered $INDEX batch. If the batch is < B,
- * the earliest entries from _live are inserted into the $INDEX until the
- * batch == B. Then, the remaining entries in _live are batched to $INDEX+1,
- * $INDEX+2, and so on. The easiest way to visualise this is that the timeline
- * goes from old to new, left to right:
- *          -2         -1         0         1
- * <--OLD---------------------------------------NEW-->
- *        [a,b,c]    [d,e,f]   [g,h,i]   [j,k,l]
- *
- * Purging
- * -------
- * Events from the timeline can be purged by removing the lowest
- * timeline_$INDEX in the store.
- *
- * Example
- * -------
- * A room with room_id !foo:bar has 9 messages (M1->9 where 9=newest) with a
- * batch size of 4. The very first time, there is no entry for !foo:bar until
- * storeRoom() is called, which results in the keys: [Full Insert]
- *   room_!foo:bar_timeline_0 : [M1, M2, M3, M4]
- *   room_!foo:bar_timeline_1 : [M5, M6, M7, M8]
- *   room_!foo:bar_timeline_2 : [M9]
- *   room_!foo:bar_state: { ... }
- *
- * 5 new messages (N1-5, 5=newest) arrive and are then added: [Incremental Insert]
- *   room_!foo:bar_timeline_live: [N1]
- *   room_!foo:bar_timeline_live: [N1, N2]
- *   room_!foo:bar_timeline_live: [N1, N2, N3]
- *   room_!foo:bar_timeline_live: [N1, N2, N3, N4]
- *   room_!foo:bar_timeline_live: [N1, N2, N3, N4, N5]
- *
- * App is shutdown. Restarts. The timeline is synced [Timeline Sync]
- *   room_!foo:bar_timeline_2 : [M9, N1, N2, N3]
- *   room_!foo:bar_timeline_3 : [N4, N5]
- *   room_!foo:bar_timeline_live: []
- *
- * And the room is retrieved with 8 messages: [Room Retrieval]
- *   Room.timeline: [M7, M8, M9, N1, N2, N3, N4, N5]
- *   Room.storageToken: => early_index = 1 because that's where M7 is.
- *
- * 3 earlier messages are requested: [Earlier retrieval]
- *   Use storageToken to find batch index 1. Scan batch for earliest event ID.
- *   earliest event = M7
- *   events = room_!foo:bar_timeline_1 where event < M7 = [M5, M6]
- * Too few events, use next index (0) and get 1 more:
- *   events = room_!foo:bar_timeline_0 = [M1, M2, M3, M4] => [M4]
- * Return concatentation:
- *   [M4, M5, M6]
- *
- * Purge oldest events: [Purge]
- *   del room_!foo:bar_timeline_0
- *
- * @module store/webstorage - */ -var DEBUG = false; // set true to enable console logging. -var utils = require("../utils"); -var Room = require("../models/room"); -var User = require("../models/user"); -var MatrixEvent = require("../models/event").MatrixEvent; - -/** - * Construct a web storage store, capable of storing rooms and users. - * @constructor - * @param {WebStorage} webStore A web storage implementation, e.g. - * 'window.localStorage' or 'window.sessionStorage' or a custom implementation. - * @param {integer} batchSize The number of events to store per key/value (room - * scoped). Use -1 to store all events for a room under one key/value. - * @throws if the supplied 'store' does not meet the Storage interface of the - * WebStorage API. - */ -function WebStorageStore(webStore, batchSize) { - this.store = webStore; - this.batchSize = batchSize; - if (!utils.isFunction(webStore.getItem) || !utils.isFunction(webStore.setItem) || - !utils.isFunction(webStore.removeItem) || !utils.isFunction(webStore.key)) { - throw new Error( - "Supplied webStore does not meet the WebStorage API interface" - ); - } - if (!parseInt(webStore.length) && webStore.length !== 0) { - throw new Error( - "Supplied webStore does not meet the WebStorage API interface (length)" - ); - } - // cached list of room_ids this is storing. - this._roomIds = []; - this._syncedWithStore = false; - // tokens used to remember which index the room instance is at. - this._tokens = [ - // { earliestIndex: -4 } - ]; -} - - -/** - * Retrieve the token to stream from. - * @return {string} The token or null. - */ -WebStorageStore.prototype.getSyncToken = function() { - return this.store.getItem("sync_token"); -}; - -/** - * Set the token to stream from. - * @param {string} token The token to stream from. - */ -WebStorageStore.prototype.setSyncToken = function(token) { - this.store.setItem("sync_token", token); -}; - -/** - * Store a room in web storage. - * @param {Room} room - */ -WebStorageStore.prototype.storeRoom = function(room) { - var serRoom = SerialisedRoom.fromRoom(room, this.batchSize); - persist(this.store, serRoom); - if (this._roomIds.indexOf(room.roomId) === -1) { - this._roomIds.push(room.roomId); - } -}; - -/** - * Retrieve a room from web storage. - * @param {string} roomId - * @return {?Room} - */ -WebStorageStore.prototype.getRoom = function(roomId) { - // probe if room exists; break early if not. Every room should have state. - if (!getItem(this.store, keyName(roomId, "state"))) { - debuglog("getRoom: No room with id %s found.", roomId); - return null; - } - var timelineKeys = getTimelineIndices(this.store, roomId); - if (timelineKeys.indexOf("live") !== -1) { - debuglog("getRoom: Live events found. Syncing timeline for %s", roomId); - this._syncTimeline(roomId, timelineKeys); - } - return loadRoom(this.store, roomId, this.batchSize, this._tokens); -}; - -/** - * Get a list of all rooms from web storage. - * @return {Array} An empty array. - */ -WebStorageStore.prototype.getRooms = function() { - var rooms = []; - var i; - if (!this._syncedWithStore) { - // sync with the store to set this._roomIds correctly. We know there is - // exactly one 'state' key for each room, so we grab them. - this._roomIds = []; - for (i = 0; i < this.store.length; i++) { - if (this.store.key(i).indexOf("room_") === 0 && - this.store.key(i).indexOf("_state") !== -1) { - // grab the middle bit which is the room ID - var k = this.store.key(i); - this._roomIds.push( - k.substring("room_".length, k.length - "_state".length) - ); - } - } - this._syncedWithStore = true; - } - // call getRoom on each room_id - for (i = 0; i < this._roomIds.length; i++) { - var rm = this.getRoom(this._roomIds[i]); - if (rm) { - rooms.push(rm); - } - } - return rooms; -}; - -/** - * Get a list of summaries from web storage. - * @return {Array} An empty array. - */ -WebStorageStore.prototype.getRoomSummaries = function() { - return []; -}; - -/** - * Store a user in web storage. - * @param {User} user - */ -WebStorageStore.prototype.storeUser = function(user) { - // persist the events used to make the user, we can reconstruct on demand. - setItem(this.store, "user_" + user.userId, { - presence: user.events.presence ? user.events.presence.event : null - }); -}; - -/** - * Get a user from web storage. - * @param {string} userId - * @return {User} - */ -WebStorageStore.prototype.getUser = function(userId) { - var userData = getItem(this.store, "user_" + userId); - if (!userData) { - return null; - } - var user = new User(userId); - if (userData.presence) { - user.setPresenceEvent(new MatrixEvent(userData.presence)); - } - return user; -}; - -/** - * Retrieve scrollback for this room. Automatically adds events to the timeline. - * @param {Room} room The matrix room to add the events to the start of the timeline. - * @param {integer} limit The max number of old events to retrieve. - * @return {Array} An array of objects which will be at most 'limit' - * length and at least 0. The objects are the raw event JSON. The last element - * is the 'oldest' (for parity with homeserver scrollback APIs). - */ -WebStorageStore.prototype.scrollback = function(room, limit) { - if (room.storageToken === undefined || room.storageToken >= this._tokens.length) { - return []; - } - // find the index of the earliest event in this room's timeline - var storeData = this._tokens[room.storageToken] || {}; - var i; - var earliestIndex = storeData.earliestIndex; - var earliestEventId = room.timeline[0] ? room.timeline[0].getId() : null; - debuglog( - "scrollback in %s (timeline=%s msgs) i=%s, timeline[0].id=%s - req %s events", - room.roomId, room.timeline.length, earliestIndex, earliestEventId, limit - ); - var batch = getItem( - this.store, keyName(room.roomId, "timeline", earliestIndex) - ); - if (!batch) { - // bad room or already at start, either way we have nothing to give. - debuglog("No batch with index %s found.", earliestIndex); - return []; - } - // populate from this batch first - var scrollback = []; - var foundEventId = false; - for (i = batch.length - 1; i >= 0; i--) { - // go back and find the earliest event ID, THEN start adding entries. - // Make a MatrixEvent so we don't assume .event_id exists - // (e.g v2/v3 JSON may be different) - var matrixEvent = new MatrixEvent(batch[i]); - if (matrixEvent.getId() === earliestEventId) { - foundEventId = true; - debuglog( - "Found timeline[0] event at position %s in batch %s", - i, earliestIndex - ); - continue; - } - if (!foundEventId) { - continue; - } - // add entry - debuglog("Add event at position %s in batch %s", i, earliestIndex); - scrollback.push(batch[i]); - if (scrollback.length === limit) { - break; - } - } - if (scrollback.length === limit) { - debuglog("Batch has enough events to satisfy request."); - return scrollback; - } - if (!foundEventId) { - // the earliest index batch didn't contain the event. In other words, - // this timeline is at a state we don't know, so bail. - debuglog( - "Failed to find event ID %s in batch %s", earliestEventId, earliestIndex - ); - return []; - } - - // get the requested earlier events from earlier batches - while (scrollback.length < limit) { - earliestIndex--; - batch = getItem( - this.store, keyName(room.roomId, "timeline", earliestIndex) - ); - if (!batch) { - // no more events - debuglog("No batch found at index %s", earliestIndex); - break; - } - for (i = batch.length - 1; i >= 0; i--) { - debuglog("Add event at position %s in batch %s", i, earliestIndex); - scrollback.push(batch[i]); - if (scrollback.length === limit) { - break; - } - } - } - debuglog( - "Out of %s requested events, returning %s. New index=%s", - limit, scrollback.length, earliestIndex - ); - room.addEventsToTimeline(utils.map(scrollback, function(e) { - return new MatrixEvent(e); - }), true, room.getLiveTimeline()); - - this._tokens[room.storageToken] = { - earliestIndex: earliestIndex - }; - return scrollback; -}; - -/** - * Store events for a room. The events have already been added to the timeline. - * @param {Room} room The room to store events for. - * @param {Array} events The events to store. - * @param {string} token The token associated with these events. - * @param {boolean} toStart True if these are paginated results. The last element - * is the 'oldest' (for parity with homeserver scrollback APIs). - */ -WebStorageStore.prototype.storeEvents = function(room, events, token, toStart) { - if (toStart) { - // add paginated events to lowest batch indexes (can go -ve) - var lowIndex = getIndexExtremity( - getTimelineIndices(this.store, room.roomId), true - ); - var i, key, batch; - for (i = 0; i < events.length; i++) { // loop events to be stored - key = keyName(room.roomId, "timeline", lowIndex); - batch = getItem(this.store, key) || []; - while (batch.length < this.batchSize && i < events.length) { - batch.unshift(events[i].event); - i++; // increment to insert next event into this batch - } - i--; // decrement to avoid skipping one (for loop ++s) - setItem(this.store, key, batch); - lowIndex--; // decrement index to get a new batch. - } - } - else { - // dump as live events - var liveEvents = getItem( - this.store, keyName(room.roomId, "timeline", "live") - ) || []; - debuglog( - "Adding %s events to %s live list (which has %s already)", - events.length, room.roomId, liveEvents.length - ); - var updateState = false; - liveEvents = liveEvents.concat(utils.map(events, function(me) { - // cheeky check to avoid looping twice - if (me.isState()) { - updateState = true; - } - return me.event; - })); - setItem( - this.store, keyName(room.roomId, "timeline", "live"), liveEvents - ); - if (updateState) { - debuglog("Storing state for %s as new events updated state", room.roomId); - // use 0 batch size; we don't care about batching right now. - var serRoom = SerialisedRoom.fromRoom(room, 0); - setItem(this.store, keyName(serRoom.roomId, "state"), serRoom.state); - } - } -}; - -/** - * Sync the 'live' timeline, batching live events according to 'batchSize'. - * @param {string} roomId The room to sync the timeline. - * @param {Array} timelineIndices Optional. The indices in the timeline - * if known already. - */ -WebStorageStore.prototype._syncTimeline = function(roomId, timelineIndices) { - timelineIndices = timelineIndices || getTimelineIndices(this.store, roomId); - var liveEvents = getItem(this.store, keyName(roomId, "timeline", "live")) || []; - - // get the highest numbered $INDEX batch - var highestIndex = getIndexExtremity(timelineIndices); - var hiKey = keyName(roomId, "timeline", highestIndex); - var hiBatch = getItem(this.store, hiKey) || []; - // fill up the existing batch first. - while (hiBatch.length < this.batchSize && liveEvents.length > 0) { - hiBatch.push(liveEvents.shift()); - } - setItem(this.store, hiKey, hiBatch); - - // start adding new batches as required - var batch = []; - while (liveEvents.length > 0) { - batch.push(liveEvents.shift()); - if (batch.length === this.batchSize || liveEvents.length === 0) { - // persist the full batch and make another - highestIndex++; - hiKey = keyName(roomId, "timeline", highestIndex); - setItem(this.store, hiKey, batch); - batch = []; - } - } - // reset live array - setItem(this.store, keyName(roomId, "timeline", "live"), []); -}; - - -/** - * Store a filter. - * @param {Filter} filter - */ -WebStorageStore.prototype.storeFilter = function(filter) { -}; - -/** - * Retrieve a filter. - * @param {string} userId - * @param {string} filterId - * @return {?Filter} A filter or null. - */ -WebStorageStore.prototype.getFilter = function(userId, filterId) { - return null; -}; - -function SerialisedRoom(roomId) { - this.state = { - events: {} - }; - this.timeline = { - // $INDEX: [] - }; - this.roomId = roomId; -} - -/** - * Convert a Room instance into a SerialisedRoom instance which can be stored - * in the key value store. - * @param {Room} room The matrix room to convert - * @param {integer} batchSize The number of events per timeline batch - * @return {SerialisedRoom} A serialised room representation of 'room'. - */ -SerialisedRoom.fromRoom = function(room, batchSize) { - var self = new SerialisedRoom(room.roomId); - var index; - self.state.pagination_token = room.oldState.paginationToken; - // [room_$ROOMID_state] downcast to POJO from MatrixEvent - utils.forEach(utils.keys(room.currentState.events), function(eventType) { - utils.forEach(utils.keys(room.currentState.events[eventType]), function(skey) { - if (!self.state.events[eventType]) { - self.state.events[eventType] = {}; - } - self.state.events[eventType][skey] = ( - room.currentState.events[eventType][skey].event - ); - }); - }); - - // [room_$ROOMID_timeline_$INDEX] - if (batchSize > 0) { - index = 0; - while (index * batchSize < room.timeline.length) { - self.timeline[index] = room.timeline.slice( - index * batchSize, (index + 1) * batchSize - ); - self.timeline[index] = utils.map(self.timeline[index], function(me) { - // use POJO not MatrixEvent - return me.event; - }); - index++; - } - } - else { // don't batch - self.timeline[0] = utils.map(room.timeline, function(matrixEvent) { - return matrixEvent.event; - }); - } - return self; -}; - -function loadRoom(store, roomId, numEvents, tokenArray) { - var room = new Room(roomId, { - storageToken: tokenArray.length - }); - - // populate state (flatten nested struct to event array) - var currentStateMap = getItem(store, keyName(roomId, "state")); - var stateEvents = []; - utils.forEach(utils.keys(currentStateMap.events), function(eventType) { - utils.forEach(utils.keys(currentStateMap.events[eventType]), function(skey) { - stateEvents.push(currentStateMap.events[eventType][skey]); - }); - }); - // TODO: Fix logic dupe with MatrixClient._processRoomEvents - var oldStateEvents = utils.map( - utils.deepCopy(stateEvents), function(e) { - return new MatrixEvent(e); - } - ); - var currentStateEvents = utils.map(stateEvents, function(e) { - return new MatrixEvent(e); - } - ); - room.oldState.setStateEvents(oldStateEvents); - room.currentState.setStateEvents(currentStateEvents); - - // add most recent numEvents - var recentEvents = []; - var index = getIndexExtremity(getTimelineIndices(store, roomId)); - var eventIndex = index; - var i, key, batch; - while (recentEvents.length < numEvents) { - key = keyName(roomId, "timeline", index); - batch = getItem(store, key) || []; - if (batch.length === 0) { - // nothing left in the store. - break; - } - for (i = batch.length - 1; i >= 0; i--) { - recentEvents.unshift(new MatrixEvent(batch[i])); - if (recentEvents.length === numEvents) { - eventIndex = index; - break; - } - } - index--; - } - // add events backwards to diverge old state correctly. - room.addEventsToTimeline(recentEvents.reverse(), true, room.getLiveTimeline()); - room.oldState.paginationToken = currentStateMap.pagination_token; - // set the token data to let us know which index this room instance is at - // for scrollback. - tokenArray.push({ - earliestIndex: eventIndex - }); - return room; -} - -function persist(store, serRoom) { - setItem(store, keyName(serRoom.roomId, "state"), serRoom.state); - utils.forEach(utils.keys(serRoom.timeline), function(index) { - setItem(store, - keyName(serRoom.roomId, "timeline", index), - serRoom.timeline[index] - ); - }); -} - -function getTimelineIndices(store, roomId) { - var keys = []; - for (var i = 0; i < store.length; i++) { - if (store.key(i).indexOf(keyName(roomId, "timeline_")) !== -1) { - // e.g. room_$ROOMID_timeline_0 => 0 - keys.push( - store.key(i).replace(keyName(roomId, "timeline_"), "") - ); - } - } - return keys; -} - -function getIndexExtremity(timelineIndices, getLowest) { - var extremity, index; - for (var i = 0; i < timelineIndices.length; i++) { - index = parseInt(timelineIndices[i]); - if (!isNaN(index) && ( - extremity === undefined || - !getLowest && index > extremity || - getLowest && index < extremity)) { - extremity = index; - } - } - return extremity; -} - -function keyName(roomId, key, index) { - return "room_" + roomId + "_" + key + ( - index === undefined ? "" : ("_" + index) - ); -} - -function getItem(store, key) { - try { - return JSON.parse(store.getItem(key)); - } - catch (e) { - debuglog("Failed to get key %s: %s", key, e); - debuglog(e.stack); - } - return null; -} - -function setItem(store, key, val) { - store.setItem(key, JSON.stringify(val)); -} - -function debuglog() { - if (DEBUG) { - console.log.apply(console, arguments); - } -} - -/* -function delRoomStruct(store, roomId) { - var prefix = "room_" + roomId; - var keysToRemove = []; - for (var i = 0; i < store.length; i++) { - if (store.key(i).indexOf(prefix) !== -1) { - keysToRemove.push(store.key(i)); - } - } - utils.forEach(keysToRemove, function(key) { - store.removeItem(key); - }); -} */ - -/** Web Storage Store class. */ -module.exports = WebStorageStore; From 97e421306b5713d2f1ea38ea7410c377d3dfc395 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Jan 2017 10:47:20 +0000 Subject: [PATCH 2/4] Remove UTs --- spec/unit/webstorage.spec.js | 580 ----------------------------------- 1 file changed, 580 deletions(-) delete mode 100644 spec/unit/webstorage.spec.js diff --git a/spec/unit/webstorage.spec.js b/spec/unit/webstorage.spec.js deleted file mode 100644 index 8454f324e..000000000 --- a/spec/unit/webstorage.spec.js +++ /dev/null @@ -1,580 +0,0 @@ -"use strict"; -var sdk = require("../.."); -var WebStorageStore = sdk.WebStorageStore; -var Room = sdk.Room; -var User = sdk.User; -var utils = require("../test-utils"); - -var MockStorageApi = require("../MockStorageApi"); - -describe("WebStorageStore", function() { - var store, room; - var roomId = "!foo:bar"; - var userId = "@alice:bar"; - var mockStorageApi; - var batchNum = 3; - // web storage api keys - var prefix = "room_" + roomId + "_timeline_"; - var stateKeyName = "room_" + roomId + "_state"; - - // stored state events - var stateEventMap = { - "m.room.member": {}, - "m.room.name": {} - }; - stateEventMap["m.room.member"][userId] = utils.mkMembership( - {user: userId, room: roomId, mship: "join"} - ); - stateEventMap["m.room.name"][""] = utils.mkEvent( - {user: userId, room: roomId, type: "m.room.name", - content: { - name: "foo" - }} - ); - - beforeEach(function() { - utils.beforeEach(this); // eslint-disable-line no-invalid-this - mockStorageApi = new MockStorageApi(); - store = new WebStorageStore(mockStorageApi, batchNum); - room = new Room(roomId); - }); - - describe("constructor", function() { - it("should throw if the WebStorage API functions are missing", function() { - expect(function() { - store = new WebStorageStore({}, 5); - }).toThrow(); - expect(function() { - mockStorageApi.length = undefined; - store = new WebStorageStore(mockStorageApi, 5); - }).toThrow(); - }); - }); - - describe("syncToken", function() { - it("get: should return the token from the store", function() { - var token = "flibble"; - store.setSyncToken(token); - expect(store.getSyncToken()).toEqual(token); - expect(mockStorageApi.length).toEqual(1); - }); - it("get: should return null if the token does not exist", function() { - expect(store.getSyncToken()).toEqual(null); - expect(mockStorageApi.length).toEqual(0); - }); - }); - - describe("storeRoom", function() { - it("should persist the room state correctly", function() { - var stateEvents = [ - utils.mkEvent({ - event: true, type: "m.room.create", user: userId, room: roomId, - content: { - creator: userId - } - }), - utils.mkMembership({ - event: true, user: userId, room: roomId, mship: "join" - }) - ]; - room.currentState.setStateEvents(stateEvents); - store.storeRoom(room); - var storedEvents = getItem(mockStorageApi, - "room_" + roomId + "_state" - ).events; - expect(storedEvents["m.room.create"][""]).toEqual(stateEvents[0].event); - }); - - it("should persist timeline events correctly", function() { - var timelineEvents = []; - var entries = batchNum + batchNum - 1; - var i = 0; - for (i = 0; i < entries; i++) { - timelineEvents.push( - utils.mkMessage({room: roomId, user: userId, event: true}) - ); - } - room.timeline = timelineEvents; - store.storeRoom(room); - expect(getItem(mockStorageApi, prefix + "-1")).toBe(null); - expect(getItem(mockStorageApi, prefix + "2")).toBe(null); - expect(getItem(mockStorageApi, prefix + "live")).toBe(null); - var timeline0 = getItem(mockStorageApi, prefix + "0"); - var timeline1 = getItem(mockStorageApi, prefix + "1"); - expect(timeline0.length).toEqual(batchNum); - expect(timeline1.length).toEqual(batchNum - 1); - for (i = 0; i < batchNum; i++) { - expect(timeline0[i]).toEqual(timelineEvents[i].event); - if ((i + batchNum) < timelineEvents.length) { - expect(timeline1[i]).toEqual(timelineEvents[i + batchNum].event); - } - } - }); - - it("should persist timeline events in one bucket if batchNum=0", function() { - store = new WebStorageStore(mockStorageApi, 0); - var timelineEvents = []; - var entries = batchNum + batchNum - 1; - var i = 0; - for (i = 0; i < entries; i++) { - timelineEvents.push( - utils.mkMessage({room: roomId, user: userId, event: true}) - ); - } - room.timeline = timelineEvents; - store.storeRoom(room); - expect(getItem(mockStorageApi, prefix + "-1")).toBe(null); - expect(getItem(mockStorageApi, prefix + "1")).toBe(null); - expect(getItem(mockStorageApi, prefix + "live")).toBe(null); - var timeline = getItem(mockStorageApi, prefix + "0"); - expect(timeline.length).toEqual(timelineEvents.length); - for (i = 0; i < timeline.length; i++) { - expect(timeline[i]).toEqual( - timelineEvents[i].event - ); - } - }); - }); - - describe("getRoom", function() { - // stored timeline events - var timeline0, timeline1, i; - - beforeEach(function() { - timeline0 = []; - timeline1 = []; - for (i = 0; i < batchNum; i++) { - timeline1[i] = utils.mkMessage({user: userId, room: roomId}); - if (i !== (batchNum - 1)) { // miss last one - timeline0[i] = utils.mkMessage({user: userId, room: roomId}); - } - } - }); - - it("should reconstruct room state", function() { - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - - var storedRoom = store.getRoom(roomId); - expect( - storedRoom.currentState.getStateEvents("m.room.name", "").event - ).toEqual(stateEventMap["m.room.name"][""]); - expect( - storedRoom.currentState.getStateEvents("m.room.member", userId).event - ).toEqual(stateEventMap["m.room.member"][userId]); - }); - - it("should reconstruct old room state", function() { - var inviteEvent = utils.mkMembership({ - user: userId, room: roomId, mship: "invite" - }); - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", [inviteEvent]); - - var storedRoom = store.getRoom(roomId); - expect( - storedRoom.currentState.getStateEvents("m.room.member", userId).event - ).toEqual(stateEventMap["m.room.member"][userId]); - expect( - storedRoom.oldState.getStateEvents("m.room.member", userId).event - ).toEqual(inviteEvent); - }); - - it("should reconstruct the room timeline", function() { - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", timeline0); - setItem(mockStorageApi, prefix + "1", timeline1); - - var storedRoom = store.getRoom(roomId); - expect(storedRoom).not.toBeNull(); - // should only get up to the batch num timeline events - expect(storedRoom.timeline.length).toEqual(batchNum); - var timeline = timeline0.concat(timeline1); - for (i = 0; i < batchNum; i++) { - expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual( - timeline[timeline.length - 1 - i] - ); - } - }); - - it("should sync the timeline for 'live' events " + - "(full hi batch; 1+bit live batches)", function() { - // 1 and a bit events go into _live - var timelineLive = []; - timelineLive.push(utils.mkMessage({user: userId, room: roomId})); - for (i = 0; i < batchNum; i++) { - timelineLive.push( - utils.mkMessage({user: userId, room: roomId}) - ); - } - - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", timeline0); - setItem(mockStorageApi, prefix + "1", timeline1); - setItem(mockStorageApi, - // deep copy the timeline via parse/stringify else items will - // be shift()ed from timelineLive and we can't compare! - prefix + "live", JSON.parse(JSON.stringify(timelineLive)) - ); - - var storedRoom = store.getRoom(roomId); - expect(storedRoom).not.toBeNull(); - // should only get up to the batch num timeline events (highest - // index of timelineLive is the newest message) - expect(storedRoom.timeline.length).toEqual(batchNum); - for (i = 0; i < batchNum; i++) { - expect(storedRoom.timeline[i].event).toEqual( - timelineLive[i + 1] - ); - } - }); - - it("should sync the timeline for 'live' events " + - "(no low batch; 1 live batches)", function() { - var timelineLive = []; - for (i = 0; i < batchNum; i++) { - timelineLive.push( - utils.mkMessage({user: userId, room: roomId}) - ); - } - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", []); - setItem(mockStorageApi, - // deep copy the timeline via parse/stringify else items will - // be shift()ed from timelineLive and we can't compare! - prefix + "live", JSON.parse(JSON.stringify(timelineLive)) - ); - - var storedRoom = store.getRoom(roomId); - expect(storedRoom).not.toBeNull(); - // should only get up to the batch num timeline events (highest - // index of timelineLive is the newest message) - expect(storedRoom.timeline.length).toEqual(batchNum); - for (i = 0; i < batchNum; i++) { - expect(storedRoom.timeline[i].event).toEqual( - timelineLive[i] - ); - } - }); - - it("should be able to reconstruct the timeline with negative indices", - function() { - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "-5", timeline0); - setItem(mockStorageApi, prefix + "-4", timeline1); - var timeline = timeline0.concat(timeline1); - var storedRoom = store.getRoom(roomId); - expect(storedRoom).not.toBeNull(); - // should only get up to the batch num timeline events - expect(storedRoom.timeline.length).toEqual(batchNum); - for (i = 0; i < batchNum; i++) { - expect(storedRoom.timeline[batchNum - 1 - i].event).toEqual( - timeline[timeline.length - 1 - i] - ); - } - }); - - it("should return null if the room doesn't exist", function() { - expect(store.getRoom("nothing")).toEqual(null); - }); - - it("should assign a storageToken to the Room", function() { - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", timeline0); - setItem(mockStorageApi, prefix + "1", timeline1); - - var storedRoom = store.getRoom(roomId); - expect(storedRoom.storageToken).toBeDefined(); - }); - }); - - describe("scrollback", function() { - // stored timeline events - var timeline0, timeline1, timeline2; - - beforeEach(function() { - // batch size is 3 - store = new WebStorageStore(mockStorageApi, 3); - timeline0 = [ - // _ - utils.mkMessage({user: userId, room: roomId}), // 1 OLDEST - utils.mkMessage({user: userId, room: roomId}) // 2 - ]; - timeline1 = [ - utils.mkMessage({user: userId, room: roomId}), // 3 - utils.mkMessage({user: userId, room: roomId}), // 4 - utils.mkMessage({user: userId, room: roomId}) // 5 - ]; - timeline2 = [ - utils.mkMessage({user: userId, room: roomId}), // 6 - utils.mkMessage({user: userId, room: roomId}), // 7 - utils.mkMessage({user: userId, room: roomId}) // 8 NEWEST - ]; - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", timeline0); - setItem(mockStorageApi, prefix + "1", timeline1); - setItem(mockStorageApi, prefix + "2", timeline2); - }); - - it("should scroll back locally giving 'limit' events", function() { - var storedRoom = store.getRoom(roomId); - expect(storedRoom.timeline.length).toEqual(3); - var events = store.scrollback(storedRoom, 3); - expect(events.length).toEqual(3); - expect(events.reverse()).toEqual(timeline1); - }); - - it("should give less than 'limit' events near the end of the stored timeline", - function() { - var storedRoom = store.getRoom(roomId); - expect(storedRoom.timeline.length).toEqual(3); - var events = store.scrollback(storedRoom, 7); - expect(events.length).toEqual(5); - expect(events.reverse()).toEqual(timeline0.concat(timeline1)); - }); - - it("should progressively give older messages the more times scrollback is called", - function() { - var events; - var storedRoom = store.getRoom(roomId); - expect(storedRoom.timeline.length).toEqual(3); - - events = store.scrollback(storedRoom, 2); - expect(events.reverse()).toEqual([timeline1[1], timeline1[2]]); - expect(storedRoom.timeline.length).toEqual(5); - - events = store.scrollback(storedRoom, 2); - expect(events.reverse()).toEqual([timeline0[1], timeline1[0]]); - expect(storedRoom.timeline.length).toEqual(7); - - events = store.scrollback(storedRoom, 2); - expect(events).toEqual([timeline0[0]]); - expect(storedRoom.timeline.length).toEqual(8); - - events = store.scrollback(storedRoom, 2); - expect(events).toEqual([]); - expect(storedRoom.timeline.length).toEqual(8); - }); - - it("should give 0 events if there is no token on the room", function() { - var r = new Room(roomId); - expect(store.scrollback(r, 3)).toEqual([]); - }); - - it("should give 0 events for unknown rooms", function() { - var r = new Room("!unknown:room"); - r.storageToken = "foo"; - expect(store.scrollback(r, 3)).toEqual([]); - }); - - it("should give 0 events if the boundary event is the last in the timeline", - function() { - var events; - var storedRoom = store.getRoom(roomId); - expect(storedRoom.timeline.length).toEqual(3); - - // go up to the boundary (8 messages total) - events = store.scrollback(storedRoom, 5); - expect(events.length).toEqual(5); - - events = store.scrollback(storedRoom, 5); - expect(events.length).toEqual(0); - }); - }); - - describe("storeEvents", function() { - var timeline0, i; - - beforeEach(function() { - timeline0 = []; - for (i = 0; i < batchNum; i++) { - timeline0.push(utils.mkMessage({user: userId, room: roomId})); - } - setItem(mockStorageApi, stateKeyName, { - events: stateEventMap, - pagination_token: "tok" - }); - setItem(mockStorageApi, prefix + "0", timeline0); - }); - - it("should add to the live batch", function() { - var events = [ - utils.mkMessage({user: userId, room: roomId, event: true}), - utils.mkMessage({user: userId, room: roomId, event: true}) - ]; - store.storeEvents(room, events, "atoken"); - var liveEvents = getItem(mockStorageApi, prefix + "live"); - expect(liveEvents.length).toEqual(2); - expect(liveEvents[0]).toEqual(events[0].event); - expect(liveEvents[1]).toEqual(events[1].event); - }); - - it("should preserve existing live events in the store", function() { - var existingEvent = utils.mkMessage({user: userId, room: roomId}); - setItem(mockStorageApi, prefix + "live", [existingEvent]); - var events = [ - utils.mkMessage({user: userId, room: roomId, event: true}), - utils.mkMessage({user: userId, room: roomId, event: true}) - ]; - store.storeEvents(room, events, "atoken"); - var liveEvents = getItem(mockStorageApi, prefix + "live"); - expect(liveEvents.length).toEqual(3); - expect(liveEvents[0]).toEqual(existingEvent); - expect(liveEvents[1]).toEqual(events[0].event); - expect(liveEvents[2]).toEqual(events[1].event); - }); - - it("should add to the lowest batch index if toStart=true", function() { - var events = [ - utils.mkMessage({user: userId, room: roomId, event: true}), - utils.mkMessage({user: userId, room: roomId, event: true}) - ]; - store.storeEvents(room, events, "atoken", true); - var timelineNeg1 = getItem(mockStorageApi, prefix + "-1"); - expect(timelineNeg1.length).toEqual(2); - expect(timelineNeg1[0]).toEqual(events[1].event); - expect(timelineNeg1[1]).toEqual(events[0].event); - }); - - it("should add multiple batches to the lowest batch index if toStart=true", - function() { - var timelineNeg1 = []; - var timelineNeg2 = []; - for (i = 0; i < batchNum; i++) { - timelineNeg1.push( - utils.mkMessage({user: userId, room: roomId, event: true}) - ); - timelineNeg2.push( - utils.mkMessage({user: userId, room: roomId, event: true}) - ); - } - - var events = timelineNeg2.concat(timelineNeg1).reverse(); - store.storeEvents(room, events, "atoken", true); - - var storedNeg1 = getItem(mockStorageApi, prefix + "-1"); - var storedNeg2 = getItem(mockStorageApi, prefix + "-2"); - expect(timelineNeg1.length).toEqual(storedNeg1.length); - expect(timelineNeg2.length).toEqual(storedNeg2.length); - for (i = 0; i < timelineNeg1.length; i++) { - expect(timelineNeg1[i].event).toEqual(storedNeg1[i]); - expect(timelineNeg2[i].event).toEqual(storedNeg2[i]); - } - }); - - it("should update stored state if state events exist", function() { - var events = [ - utils.mkEvent({ - user: userId, room: roomId, type: "m.room.name", event: true, - content: { - name: "Room Name Here for updates" - } - }) - ]; - room.currentState.setStateEvents(events); - store.storeEvents(room, events, "atoken"); - - var liveEvents = getItem(mockStorageApi, prefix + "live"); - expect(liveEvents.length).toEqual(1); - expect(liveEvents[0]).toEqual(events[0].event); - - var stateEvents = getItem(mockStorageApi, stateKeyName); - expect(stateEvents.events["m.room.name"][""]).toEqual(events[0].event); - }); - }); - - describe("getRooms", function() { - var mkState = function(id) { - return [ - utils.mkEvent({ - event: true, type: "m.room.create", user: userId, room: id, - content: { - creator: userId - } - }), - utils.mkMembership({ - event: true, user: userId, room: id, mship: "join" - }) - ]; - }; - - it("should get all rooms in the store", function() { - var roomIds = [ - "!alpha:bet", "!beta:fet" - ]; - // store 2 dynamically - var roomA = new Room(roomIds[0]); - roomA.currentState.setStateEvents(mkState(roomIds[0])); - var roomB = new Room(roomIds[1]); - roomB.currentState.setStateEvents(mkState(roomIds[1])); - store.storeRoom(roomA); - store.storeRoom(roomB); - - var rooms = store.getRooms(); - expect(rooms.length).toEqual(2); - for (var i = 0; i < rooms.length; i++) { - var index = roomIds.indexOf(rooms[i].roomId); - expect(index).not.toEqual( - -1, "Unknown room" - ); - roomIds.splice(index, 1); - } - }); - }); - - describe("getUser", function() { - it("should be able to retrieve a stored user", function() { - var user = new User(userId); - store.storeUser(user); - var result = store.getUser(userId); - expect(result).toBeDefined(); - expect(result.userId).toEqual(userId); - }); - - it("should be able to retrieve a stored user with name data", function() { - var presence = utils.mkEvent({ - type: "m.presence", event: true, content: { - user_id: userId, - displayname: "Flibble" - } - }); - var user = new User(userId); - user.setPresenceEvent(presence); - store.storeUser(user); - var result = store.getUser(userId); - console.log(result); - expect(result.events.presence).toEqual(presence); - }); - }); -}); - -function getItem(store, key) { - return JSON.parse(store.getItem(key)); -} - -function setItem(store, key, val) { - store.setItem(key, JSON.stringify(val)); -} From 9633532dd42f979a07c195f46a8929c74d36cc23 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Jan 2017 11:57:33 +0000 Subject: [PATCH 3/4] Bump to node 6 to fix 'SyntaxError: Block-scoped declarations (let, const, function, class) not yet supported outside strict mode' --- jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins.sh b/jenkins.sh index 38f1ef9a6..d9c47a3f6 100755 --- a/jenkins.sh +++ b/jenkins.sh @@ -2,7 +2,7 @@ export NVM_DIR="/home/jenkins/.nvm" [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" -nvm use 4 +nvm use 6 npm install RC=0 From 8c52870e07785ca7bb85246e628f7132d7afa08a Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Fri, 13 Jan 2017 12:03:44 +0000 Subject: [PATCH 4/4] Dummy commit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 41202b938..c51ca5b2d 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Later versions of the SDK will: Usage ===== + Conventions -----------