More efficient syncing

Thank you @PotOfCoffee2Go I ended up taking some of your code from #8101 to get this up and running. There's still some stuff missing (like the tests!) but it gets things moving.

plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js

@@ -14,13 +14,14 @@ A sync adaptor module for synchronising with MultiWikiServer-compatible servers
 
 var CONFIG_HOST_TIDDLER = "$:/config/multiwikiclient/host",
 	DEFAULT_HOST_TIDDLER = "$protocol$//$host$/",
-	BAG_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/bag",
+	BAG_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/bag",
-	REVISION_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/revision";
+	REVISION_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/revision";
 
 function MultiWikiClientAdaptor(options) {
 	this.wiki = options.wiki;
 	this.host = this.getHost();
 	this.recipe = this.wiki.getTiddlerText("$:/config/multiwikiclient/recipe");
+	this.last_known_tiddler_id = $tw.utils.parseNumber(this.wiki.getTiddlerText("$:/state/multiwikiclient/recipe/last_tiddler_id","0"));
 	this.logger = new $tw.utils.Logger("MultiWikiClientAdaptor");
 	this.isLoggedIn = false;
 	this.isReadOnly = false;
@@ -68,6 +69,10 @@ MultiWikiClientAdaptor.prototype.getTiddlerInfo = function(tiddler) {
 	}
 };
 
+MultiWikiClientAdaptor.prototype.getTiddlerBag = function(title) {
+	return this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER,title);
+};
+
 MultiWikiClientAdaptor.prototype.getTiddlerRevision = function(title) {
 	return this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER,title);
 };
@@ -163,23 +168,38 @@ MultiWikiClientAdaptor.prototype.getCsrfToken = function() {
 };
 
 /*
-Get an array of skinny tiddler fields from the server
+Get details of changed tiddlers from the server
 */
-MultiWikiClientAdaptor.prototype.getSkinnyTiddlers = function(callback) {
+MultiWikiClientAdaptor.prototype.getUpdatedTiddlers = function(syncer,callback) {
 	var self = this;
 	$tw.utils.httpRequest({
 		url: this.host + "recipes/" + this.recipe + "/tiddlers.json",
 		data: {
-			filter: "[all[tiddlers]] -[[$:/isEncrypted]] -[prefix[$:/temp/]] -[prefix[$:/status/]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] -[[$:/library/sjcl.js]] -[[$:/core]]"
+			last_known_tiddler_id: this.last_known_tiddler_id
 		},
 		callback: function(err,data) {
 			// Check for errors
 			if(err) {
 				return callback(err);
 			}
-			var tiddlers = $tw.utils.parseJSONSafe(data);
+			var modifications = [],
-			// Invoke the callback with the skinny tiddlers
+				deletions = [];
-			callback(null,tiddlers);
+			var tiddlerInfoArray = $tw.utils.parseJSONSafe(data);
+			$tw.utils.each(tiddlerInfoArray,function(tiddlerInfo) {
+				if(tiddlerInfo.tiddler_id > self.last_known_tiddler_id) {
+					self.last_known_tiddler_id = tiddlerInfo.tiddler_id;
+				}
+				if(tiddlerInfo.is_deleted) {
+					deletions.push(tiddlerInfo.title);
+				} else {
+					modifications.push(tiddlerInfo.title);
+				}
+			});
+			// Invoke the callback with the results
+			callback(null,{
+				modifications: modifications,
+				deletions: deletions
+			});
 			// If Browswer Storage tiddlers were cached on reloading the wiki, add them after sync from server completes in the above callback.
 			if($tw.browserStorage && $tw.browserStorage.isEnabled()) { 
 				$tw.browserStorage.addCachedTiddlers();
@@ -252,7 +272,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options
 		return callback(null);
 	}
 	// If we don't have a bag it means that the tiddler hasn't been seen by the server, so we don't need to delete it
-	var bag = options.tiddlerInfo.adaptorInfo && options.tiddlerInfo.adaptorInfo.bag;
+	var bag = this.getTiddlerBag(title);
 	if(!bag) {
 		return callback(null,options.tiddlerInfo.adaptorInfo);
 	}
@@ -264,6 +284,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options
 			if(err) {
 				return callback(err);
 			}
+			self.removeTiddlerInfo(title);
 			// Invoke the callback & return null adaptorInfo
 			callback(null,null);
 		}

plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js

@@ -20,17 +20,14 @@ exports.handler = function(request,response,state) {
 	// Get the  parameters
 	var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]);
 	if(recipe_name) {
-		// Get the tiddlers in the recipe
+		// Get the tiddlers in the recipe, optionally since the specified last known tiddler_id
-		var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name);
+		var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name,{
-		// Get a skinny version of each tiddler
+			last_known_tiddler_id: state.queryParameters.last_known_tiddler_id
-		var tiddlers = [];
-		$tw.utils.each(recipeTiddlers,function(recipeTiddlerInfo) {
-			var tiddlerInfo = $tw.mws.store.getRecipeTiddler(recipeTiddlerInfo.title,recipe_name);
-			tiddlers.push(Object.assign({},tiddlerInfo.tiddler,{text: undefined}));
 		});
-		var text = JSON.stringify(tiddlers);
+		if(recipeTiddlers) {
-		state.sendResponse(200,{"Content-Type": "application/json"},text,"utf8");
+			state.sendResponse(200,{"Content-Type": "application/json"},JSON.stringify(recipeTiddlers),"utf8");
-		return;
+			return;
+		}
 	}
 	// Fail if something went wrong
 	response.writeHead(404);

plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js

@@ -77,6 +77,10 @@ exports.handler = function(request,response,state) {
 			title: "$:/config/multiwikiclient/recipe",
 			text: recipe_name
 		});
+		writeTiddler({
+			title: "$:/state/multiwikiclient/recipe/last_tiddler_id",
+			text: $tw.mws.store.getRecipeLastTiddlerId(recipe_name).toString()
+		});
 		response.write(template.substring(markerPos + marker.length))
 		// Finish response
 		response.end();

plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js

@@ -395,36 +395,105 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bag_name) {
 };
 
 /*
-Get the titles of the tiddlers in a recipe as {title:,tiddler_id:,bag_name:}. Returns null for recipes that do not exist
+Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist
 */
-SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name) {
+SqlTiddlerDatabase.prototype.getBagLastTiddlerId = function(bag_name) {
-	const rowsCheckRecipe = this.engine.runStatementGetAll(`
+	const row = this.engine.runStatementGet(`
-		SELECT * FROM recipes WHERE recipes.recipe_name = $recipe_name
+		SELECT tiddler_id
+		FROM tiddlers
+		WHERE bag_id IN (
+			SELECT bag_id
+			FROM bags
+			WHERE bag_name = $bag_name
+		)
+		ORDER BY tiddler_id DESC
+		LIMIT 1
 	`,{
-		$recipe_name: recipe_name
+		$bag_name: bag_name
 	});
-	if(rowsCheckRecipe.length === 0) {
+	if(row) {
+		return row.tiddler_id;
+	} else {
 		return null;
 	}
-	const rows = this.engine.runStatementGetAll(`
+};
-		SELECT title, tiddler_id, bag_name
+
-		FROM (
+/*
-			SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position
+Get the metadata of the tiddlers in a recipe as an array [{title:,tiddler_id:,bag_name:,is_deleted:}],
-			FROM bags AS b
+sorted in ascending order of tiddler_id.
-			INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
+
-			INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id
+Options include:
-			INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
+
-			WHERE r.recipe_name = $recipe_name
+limit: optional maximum number of results to return
-			AND t.is_deleted = FALSE
+last_known_tiddler_id: tiddler_id of the last known update. Only returns tiddlers that have been created, modified or deleted since
-			GROUP BY t.title
+include_deleted: boolean, defaults to false
-			ORDER BY t.title
+
-		)
+Returns null for recipes that do not exist
+*/
+SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name,options) {
+	options = options || {};
+	// Get the recipe ID
+	const rowsCheckRecipe = this.engine.runStatementGet(`
+		SELECT recipe_id FROM recipes WHERE recipes.recipe_name = $recipe_name
 	`,{
 		$recipe_name: recipe_name
 	});
+	if(!rowsCheckRecipe) {
+		return null;
+	}
+	const recipe_id = rowsCheckRecipe.recipe_id;
+	// Compose the query to get the tiddlers
+	const params = {
+		$recipe_id: recipe_id
+	}
+	if(options.limit) {
+		params.$limit = options.limit.toString();
+	}
+	if(options.last_known_tiddler_id) {
+		params.$last_known_tiddler_id = options.last_known_tiddler_id;
+	}
+	const rows = this.engine.runStatementGetAll(`
+		SELECT title, tiddler_id, is_deleted, bag_name
+		FROM (
+			SELECT t.title, t.tiddler_id, t.is_deleted, b.bag_name, MAX(rb.position) AS position
+			FROM bags AS b
+			INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
+			INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
+			WHERE rb.recipe_id = $recipe_id
+			${options.include_deleted ? "" : "AND t.is_deleted = FALSE"}
+			${options.last_known_tiddler_id ? "AND tiddler_id > $last_known_tiddler_id" : ""}
+			GROUP BY t.title
+			ORDER BY t.title, tiddler_id DESC
+			${options.limit ? "LIMIT $limit" : ""}
+		)
+	`,params);
 	return rows;
 };
 
+/*
+Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist
+*/
+SqlTiddlerDatabase.prototype.getRecipeLastTiddlerId = function(recipe_name) {
+	const row = this.engine.runStatementGet(`
+		SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position
+		FROM bags AS b
+		INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
+		INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id
+		INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
+		WHERE r.recipe_name = $recipe_name
+		GROUP BY t.title
+		ORDER BY t.tiddler_id DESC
+		LIMIT 1
+	`,{
+		$recipe_name: recipe_name
+	});
+	if(row) {
+		return row.tiddler_id;
+	} else {
+		return null;
+	}
+};
+
 SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bag_name) {
 	// Delete the fields
 	this.engine.runStatement(`

plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js

@@ -306,11 +306,25 @@ SqlTiddlerStore.prototype.getBagTiddlers = function(bag_name) {
 	return this.sqlTiddlerDatabase.getBagTiddlers(bag_name);
 };
 
+/*
+Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist
+*/
+SqlTiddlerStore.prototype.getBagLastTiddlerId = function(bag_name) {
+	return this.sqlTiddlerDatabase.getBagLastTiddlerId(bag_name);
+};
+
 /*
 Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist
 */
-SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name) {
+SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name,options) {
-	return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name);
+	return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name,options);
+};
+
+/*
+Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist
+*/
+SqlTiddlerStore.prototype.getRecipeLastTiddlerId = function(recipe_name) {
+	return this.sqlTiddlerDatabase.getRecipeLastTiddlerId(recipe_name);
 };
 
 SqlTiddlerStore.prototype.deleteAllTiddlersInBag = function(bag_name) {

plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js

@@ -74,12 +74,12 @@ function runSqlDatabaseTests(engine) {
 		});
 		// Verify what we've got
 		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ 
-			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
+			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
-			{ title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta' }
+			{ title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta', is_deleted: 0 }
 		]);
 		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([
-			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
+			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
-    		{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' }
+    		{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 }
 		]);
 		expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-rho").tiddler).toEqual({ title: "Hello There", text: "I'm in beta", tags: "four five six" });
 		expect(sqlTiddlerDatabase.getRecipeTiddler("Missing Tiddler","recipe-rho")).toEqual(null);
@@ -90,17 +90,17 @@ function runSqlDatabaseTests(engine) {
 		// Delete a tiddlers to ensure the underlying tiddler in the recipe shows through
 		sqlTiddlerDatabase.deleteTiddler("Hello There","bag-beta");
 		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([
-			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
+			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
-    		{ title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' }
+    		{ title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 }
 		]);
 		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ 
-			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
+			{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
-			{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' }
+			{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 }
 		]);
 		expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-beta")).toEqual(null);
 		sqlTiddlerDatabase.deleteTiddler("Another Tiddler","bag-alpha");
-		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' } ]);
+		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 } ]);
-		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' } ]);
+		expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 } ]);
 		// Save a recipe tiddler
 		expect(sqlTiddlerDatabase.saveRecipeTiddler({title: "More", text: "None"},"recipe-rho")).toEqual({tiddler_id: 7, bag_name: 'bag-beta'});
 		expect(sqlTiddlerDatabase.getRecipeTiddler("More","recipe-rho").tiddler).toEqual({title: "More", text: "None"});

plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js

@@ -123,7 +123,7 @@ function runSqlStoreTests(engine) {
 		expect(typeof(saveRecipeResult.tiddler_id)).toBe("number");
 		expect(saveRecipeResult.bag_name).toBe("bag-beta");
 
-		expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta"}]);
+		expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta", is_deleted: 0 }]);
 
 		var getRecipeTiddlerResult = store.getRecipeTiddler("Another Tiddler","recipe-rho");
 		expect(typeof(getRecipeTiddlerResult.tiddler_id)).toBe("number");