diff --git a/plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js b/plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js index 33b57ca9e..c93a321a4 100644 --- a/plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js +++ b/plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js @@ -14,13 +14,14 @@ A sync adaptor module for synchronising with MultiWikiServer-compatible servers var CONFIG_HOST_TIDDLER = "$:/config/multiwikiclient/host", DEFAULT_HOST_TIDDLER = "$protocol$//$host$/", - BAG_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/bag", - REVISION_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/revision"; + BAG_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/bag", + REVISION_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/revision"; function MultiWikiClientAdaptor(options) { this.wiki = options.wiki; this.host = this.getHost(); this.recipe = this.wiki.getTiddlerText("$:/config/multiwikiclient/recipe"); + this.last_known_tiddler_id = $tw.utils.parseNumber(this.wiki.getTiddlerText("$:/state/multiwikiclient/recipe/last_tiddler_id","0")); this.logger = new $tw.utils.Logger("MultiWikiClientAdaptor"); this.isLoggedIn = false; this.isReadOnly = false; @@ -68,6 +69,10 @@ MultiWikiClientAdaptor.prototype.getTiddlerInfo = function(tiddler) { } }; +MultiWikiClientAdaptor.prototype.getTiddlerBag = function(title) { + return this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER,title); +}; + MultiWikiClientAdaptor.prototype.getTiddlerRevision = function(title) { return this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER,title); }; @@ -163,23 +168,38 @@ MultiWikiClientAdaptor.prototype.getCsrfToken = function() { }; /* -Get an array of skinny tiddler fields from the server +Get details of changed tiddlers from the server */ -MultiWikiClientAdaptor.prototype.getSkinnyTiddlers = function(callback) { +MultiWikiClientAdaptor.prototype.getUpdatedTiddlers = function(syncer,callback) { var self = this; $tw.utils.httpRequest({ url: this.host + "recipes/" + this.recipe + "/tiddlers.json", data: { - filter: "[all[tiddlers]] -[[$:/isEncrypted]] -[prefix[$:/temp/]] -[prefix[$:/status/]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] -[[$:/library/sjcl.js]] -[[$:/core]]" + last_known_tiddler_id: this.last_known_tiddler_id }, callback: function(err,data) { // Check for errors if(err) { return callback(err); } - var tiddlers = $tw.utils.parseJSONSafe(data); - // Invoke the callback with the skinny tiddlers - callback(null,tiddlers); + var modifications = [], + deletions = []; + var tiddlerInfoArray = $tw.utils.parseJSONSafe(data); + $tw.utils.each(tiddlerInfoArray,function(tiddlerInfo) { + if(tiddlerInfo.tiddler_id > self.last_known_tiddler_id) { + self.last_known_tiddler_id = tiddlerInfo.tiddler_id; + } + if(tiddlerInfo.is_deleted) { + deletions.push(tiddlerInfo.title); + } else { + modifications.push(tiddlerInfo.title); + } + }); + // Invoke the callback with the results + callback(null,{ + modifications: modifications, + deletions: deletions + }); // If Browswer Storage tiddlers were cached on reloading the wiki, add them after sync from server completes in the above callback. if($tw.browserStorage && $tw.browserStorage.isEnabled()) { $tw.browserStorage.addCachedTiddlers(); @@ -252,7 +272,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options return callback(null); } // If we don't have a bag it means that the tiddler hasn't been seen by the server, so we don't need to delete it - var bag = options.tiddlerInfo.adaptorInfo && options.tiddlerInfo.adaptorInfo.bag; + var bag = this.getTiddlerBag(title); if(!bag) { return callback(null,options.tiddlerInfo.adaptorInfo); } @@ -264,6 +284,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options if(err) { return callback(err); } + self.removeTiddlerInfo(title); // Invoke the callback & return null adaptorInfo callback(null,null); } diff --git a/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js b/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js index 6812b9ded..5d1581d1e 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js @@ -20,17 +20,14 @@ exports.handler = function(request,response,state) { // Get the parameters var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]); if(recipe_name) { - // Get the tiddlers in the recipe - var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name); - // Get a skinny version of each tiddler - var tiddlers = []; - $tw.utils.each(recipeTiddlers,function(recipeTiddlerInfo) { - var tiddlerInfo = $tw.mws.store.getRecipeTiddler(recipeTiddlerInfo.title,recipe_name); - tiddlers.push(Object.assign({},tiddlerInfo.tiddler,{text: undefined})); + // Get the tiddlers in the recipe, optionally since the specified last known tiddler_id + var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name,{ + last_known_tiddler_id: state.queryParameters.last_known_tiddler_id }); - var text = JSON.stringify(tiddlers); - state.sendResponse(200,{"Content-Type": "application/json"},text,"utf8"); - return; + if(recipeTiddlers) { + state.sendResponse(200,{"Content-Type": "application/json"},JSON.stringify(recipeTiddlers),"utf8"); + return; + } } // Fail if something went wrong response.writeHead(404); diff --git a/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js b/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js index 413d99f72..ba0b64372 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js @@ -77,6 +77,10 @@ exports.handler = function(request,response,state) { title: "$:/config/multiwikiclient/recipe", text: recipe_name }); + writeTiddler({ + title: "$:/state/multiwikiclient/recipe/last_tiddler_id", + text: $tw.mws.store.getRecipeLastTiddlerId(recipe_name).toString() + }); response.write(template.substring(markerPos + marker.length)) // Finish response response.end(); diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js index 7809d6857..cae1d34d9 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js @@ -395,36 +395,105 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bag_name) { }; /* -Get the titles of the tiddlers in a recipe as {title:,tiddler_id:,bag_name:}. Returns null for recipes that do not exist +Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist */ -SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name) { - const rowsCheckRecipe = this.engine.runStatementGetAll(` - SELECT * FROM recipes WHERE recipes.recipe_name = $recipe_name +SqlTiddlerDatabase.prototype.getBagLastTiddlerId = function(bag_name) { + const row = this.engine.runStatementGet(` + SELECT tiddler_id + FROM tiddlers + WHERE bag_id IN ( + SELECT bag_id + FROM bags + WHERE bag_name = $bag_name + ) + ORDER BY tiddler_id DESC + LIMIT 1 `,{ - $recipe_name: recipe_name + $bag_name: bag_name }); - if(rowsCheckRecipe.length === 0) { + if(row) { + return row.tiddler_id; + } else { return null; } - const rows = this.engine.runStatementGetAll(` - SELECT title, tiddler_id, bag_name - FROM ( - SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position - FROM bags AS b - INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id - INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id - INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id - WHERE r.recipe_name = $recipe_name - AND t.is_deleted = FALSE - GROUP BY t.title - ORDER BY t.title - ) +}; + +/* +Get the metadata of the tiddlers in a recipe as an array [{title:,tiddler_id:,bag_name:,is_deleted:}], +sorted in ascending order of tiddler_id. + +Options include: + +limit: optional maximum number of results to return +last_known_tiddler_id: tiddler_id of the last known update. Only returns tiddlers that have been created, modified or deleted since +include_deleted: boolean, defaults to false + +Returns null for recipes that do not exist +*/ +SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name,options) { + options = options || {}; + // Get the recipe ID + const rowsCheckRecipe = this.engine.runStatementGet(` + SELECT recipe_id FROM recipes WHERE recipes.recipe_name = $recipe_name `,{ $recipe_name: recipe_name }); + if(!rowsCheckRecipe) { + return null; + } + const recipe_id = rowsCheckRecipe.recipe_id; + // Compose the query to get the tiddlers + const params = { + $recipe_id: recipe_id + } + if(options.limit) { + params.$limit = options.limit.toString(); + } + if(options.last_known_tiddler_id) { + params.$last_known_tiddler_id = options.last_known_tiddler_id; + } + const rows = this.engine.runStatementGetAll(` + SELECT title, tiddler_id, is_deleted, bag_name + FROM ( + SELECT t.title, t.tiddler_id, t.is_deleted, b.bag_name, MAX(rb.position) AS position + FROM bags AS b + INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id + INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id + WHERE rb.recipe_id = $recipe_id + ${options.include_deleted ? "" : "AND t.is_deleted = FALSE"} + ${options.last_known_tiddler_id ? "AND tiddler_id > $last_known_tiddler_id" : ""} + GROUP BY t.title + ORDER BY t.title, tiddler_id DESC + ${options.limit ? "LIMIT $limit" : ""} + ) + `,params); return rows; }; +/* +Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist +*/ +SqlTiddlerDatabase.prototype.getRecipeLastTiddlerId = function(recipe_name) { + const row = this.engine.runStatementGet(` + SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position + FROM bags AS b + INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id + INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id + INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id + WHERE r.recipe_name = $recipe_name + GROUP BY t.title + ORDER BY t.tiddler_id DESC + LIMIT 1 + `,{ + $recipe_name: recipe_name + }); + if(row) { + return row.tiddler_id; + } else { + return null; + } +}; + SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bag_name) { // Delete the fields this.engine.runStatement(` diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js index 5afee7697..6a7637d5c 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js @@ -306,11 +306,25 @@ SqlTiddlerStore.prototype.getBagTiddlers = function(bag_name) { return this.sqlTiddlerDatabase.getBagTiddlers(bag_name); }; +/* +Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist +*/ +SqlTiddlerStore.prototype.getBagLastTiddlerId = function(bag_name) { + return this.sqlTiddlerDatabase.getBagLastTiddlerId(bag_name); +}; + /* Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist */ -SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name) { - return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name); +SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name,options) { + return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name,options); +}; + +/* +Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist +*/ +SqlTiddlerStore.prototype.getRecipeLastTiddlerId = function(recipe_name) { + return this.sqlTiddlerDatabase.getRecipeLastTiddlerId(recipe_name); }; SqlTiddlerStore.prototype.deleteAllTiddlersInBag = function(bag_name) { diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js b/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js index 04744615d..d44c70693 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js @@ -74,12 +74,12 @@ function runSqlDatabaseTests(engine) { }); // Verify what we've got expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ - { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' }, - { title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta' } + { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 }, + { title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta', is_deleted: 0 } ]); expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ - { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' }, - { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' } + { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 }, + { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 } ]); expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-rho").tiddler).toEqual({ title: "Hello There", text: "I'm in beta", tags: "four five six" }); expect(sqlTiddlerDatabase.getRecipeTiddler("Missing Tiddler","recipe-rho")).toEqual(null); @@ -90,17 +90,17 @@ function runSqlDatabaseTests(engine) { // Delete a tiddlers to ensure the underlying tiddler in the recipe shows through sqlTiddlerDatabase.deleteTiddler("Hello There","bag-beta"); expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ - { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' }, - { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' } + { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 }, + { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 } ]); expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ - { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' }, - { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' } + { title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 }, + { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 } ]); expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-beta")).toEqual(null); sqlTiddlerDatabase.deleteTiddler("Another Tiddler","bag-alpha"); - expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' } ]); - expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' } ]); + expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 } ]); + expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 } ]); // Save a recipe tiddler expect(sqlTiddlerDatabase.saveRecipeTiddler({title: "More", text: "None"},"recipe-rho")).toEqual({tiddler_id: 7, bag_name: 'bag-beta'}); expect(sqlTiddlerDatabase.getRecipeTiddler("More","recipe-rho").tiddler).toEqual({title: "More", text: "None"}); diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js b/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js index ae65c7fe2..73b6f7380 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js @@ -123,7 +123,7 @@ function runSqlStoreTests(engine) { expect(typeof(saveRecipeResult.tiddler_id)).toBe("number"); expect(saveRecipeResult.bag_name).toBe("bag-beta"); - expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta"}]); + expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta", is_deleted: 0 }]); var getRecipeTiddlerResult = store.getRecipeTiddler("Another Tiddler","recipe-rho"); expect(typeof(getRecipeTiddlerResult.tiddler_id)).toBe("number");