Restructure syncer to use a single state machine

* Alternate fix for inconsistent list template syntax

First attempt, which fails on the ListWidget/WithMissingTemplate test.

* Make WithMissingTemplate test pass, inefficiently

Unfortunately, this ends up being very inefficient, because the
clone-and-mutate logic is repeated for every list item. Not ideal.

* More efficient way to do it

This also makes the failing test pass, but far more efficiently.

* Improve performance of list template discovery

Since parse tree nodes never change after widget creation (whereas
attribute values *can* change), we can safely search for the explicit
list templtaes only once, at widget creation time. This saves time as
the search doesn't have to be done on each re-render, and also allows us
to safely do a clone-and-mutate step to extract the list widget's body
(if any) without any `$list-empty` or other items. That, in turn, allows
using the list widget's body as the template even if `$list-empty` is
specified inside the widget body.

bin/ci-test.sh

@@ -2,9 +2,6 @@
 
 # test TiddlyWiki5 for tiddlywiki.com
 
-npm install playwright @playwright/test
-npx playwright install chromium firefox --with-deps
-
 node ./tiddlywiki.js \
 	./editions/test \
 	--verbose \
@@ -13,4 +10,7 @@ node ./tiddlywiki.js \
 	--test \
 	|| exit 1
 
+npm install playwright @playwright/test
+npx playwright install chromium firefox --with-deps
+
 npx playwright test

core/modules/filters/tag.js

@@ -52,7 +52,7 @@ exports.tag = function(source,operator,options) {
 			}
 		}
 	}
-	return $tw.browser ? results.reverse() : results;
+	return results;
 };
 
 })();

core/modules/syncer.js

@@ -24,7 +24,7 @@ Syncer.prototype.titleSyncPollingInterval = "$:/config/SyncPollingInterval";
 Syncer.prototype.titleSyncDisableLazyLoading = "$:/config/SyncDisableLazyLoading";
 Syncer.prototype.titleSavedNotification = "$:/language/Notifications/Save/Done";
 Syncer.prototype.titleSyncThrottleInterval = "$:/config/SyncThrottleInterval";
-Syncer.prototype.taskTimerInterval = 1 * 1000; // Interval for sync timer
+Syncer.prototype.taskTimerInterval = 0.25 * 1000; // Interval for sync timer
 Syncer.prototype.throttleInterval = 1 * 1000; // Defer saving tiddlers if they've changed in the last 1s...
 Syncer.prototype.errorRetryInterval = 5 * 1000; // Interval to retry after an error
 Syncer.prototype.fallbackInterval = 10 * 1000; // Unless the task is older than 10s
@@ -74,9 +74,11 @@ function Syncer(options) {
 	this.titlesHaveBeenLazyLoaded = {}; // Hashmap of titles of tiddlers that have already been lazily loaded from the server
 	// Timers
 	this.taskTimerId = null; // Timer for task dispatch
-	this.pollTimerId = null; // Timer for polling server
 	// Number of outstanding requests
 	this.numTasksInProgress = 0;
+	// True when we want to force an immediate sync from the server
+	this.forceSyncFromServer = false;
+	this.timestampLastSyncFromServer = new Date();
 	// Listen out for changes to tiddlers
 	this.wiki.addEventListener("change",function(changes) {
 		// Filter the changes to just include ones that are being synced
@@ -187,6 +189,7 @@ Syncer.prototype.readTiddlerInfo = function() {
 	// Record information for known tiddlers
 	var self = this,
 		tiddlers = this.getSyncedTiddlers();
+	this.logger.log("Initialising tiddlerInfo for " + tiddlers.length + " tiddlers");
 	$tw.utils.each(tiddlers,function(title) {
 		var tiddler = self.wiki.getTiddler(title);
 		if(tiddler) {
@@ -203,33 +206,38 @@ Syncer.prototype.readTiddlerInfo = function() {
 Checks whether the wiki is dirty (ie the window shouldn't be closed)
 */
 Syncer.prototype.isDirty = function() {
-	this.logger.log("Checking dirty status");
-	// Check tiddlers that are in the store and included in the filter function
-	var titles = this.getSyncedTiddlers();
-	for(var index=0; index<titles.length; index++) {
-		var title = titles[index],
-			tiddlerInfo = this.tiddlerInfo[title];
-		if(this.wiki.tiddlerExists(title)) {
-			if(tiddlerInfo) {
-				// If the tiddler is known on the server and has been modified locally then it needs to be saved to the server
-				if(this.wiki.getChangeCount(title) > tiddlerInfo.changeCount) {
+	var self = this;
+	function checkIsDirty() {
+		// Check tiddlers that are in the store and included in the filter function
+		var titles = self.getSyncedTiddlers();
+		for(var index=0; index<titles.length; index++) {
+			var title = titles[index],
+				tiddlerInfo = self.tiddlerInfo[title];
+			if(self.wiki.tiddlerExists(title)) {
+				if(tiddlerInfo) {
+					// If the tiddler is known on the server and has been modified locally then it needs to be saved to the server
+					if(self.wiki.getChangeCount(title) > tiddlerInfo.changeCount) {
+						return true;
+					}
+				} else {
+					// If the tiddler isn't known on the server then it needs to be saved to the server
 					return true;
 				}
-			} else {
-				// If the tiddler isn't known on the server then it needs to be saved to the server
+			}
+		}
+		// Check tiddlers that are known from the server but not currently in the store
+		titles = Object.keys(self.tiddlerInfo);
+		for(index=0; index<titles.length; index++) {
+			if(!self.wiki.tiddlerExists(titles[index])) {
+				// There must be a pending delete
 				return true;
 			}
 		}
+		return false;
 	}
-	// Check tiddlers that are known from the server but not currently in the store
-	titles = Object.keys(this.tiddlerInfo);
-	for(index=0; index<titles.length; index++) {
-		if(!this.wiki.tiddlerExists(titles[index])) {
-			// There must be a pending delete
-			return true;
-		}
-	}
-	return false;
+	var dirtyStatus = checkIsDirty();
+	this.logger.log("Dirty status was " + dirtyStatus);
+	return dirtyStatus;
 };
 
 /*
@@ -258,6 +266,7 @@ Syncer.prototype.storeTiddler = function(tiddlerFields) {
 		adaptorInfo: this.syncadaptor.getTiddlerInfo(tiddler),
 		changeCount: this.wiki.getChangeCount(tiddlerFields.title)
 	};
+	this.logger.log("Updating tiddler info in syncer.storeTiddler for " + tiddlerFields.title + " " + JSON.stringify(this.tiddlerInfo[tiddlerFields.title]));
 };
 
 Syncer.prototype.getStatus = function(callback) {
@@ -293,90 +302,8 @@ Syncer.prototype.getStatus = function(callback) {
 Synchronise from the server by reading the skinny tiddler list and queuing up loads for any tiddlers that we don't already have up to date
 */
 Syncer.prototype.syncFromServer = function() {
-	var self = this,
-		cancelNextSync = function() {
-			if(self.pollTimerId) {
-				clearTimeout(self.pollTimerId);
-				self.pollTimerId = null;
-			}
-		},
-		triggerNextSync = function() {
-			self.pollTimerId = setTimeout(function() {
-				self.pollTimerId = null;
-				self.syncFromServer.call(self);
-			},self.pollTimerInterval);
-		},
-		syncSystemFromServer = (self.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes" ? true : false);
-	if(this.syncadaptor && this.syncadaptor.getUpdatedTiddlers) {
-		this.logger.log("Retrieving updated tiddler list");
-		cancelNextSync();
-		this.syncadaptor.getUpdatedTiddlers(self,function(err,updates) {
-			triggerNextSync();
-			if(err) {
-				self.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
-				return;
-			}
-			if(updates) {
-				$tw.utils.each(updates.modifications,function(title) {
-					self.titlesToBeLoaded[title] = true;
-				});
-				$tw.utils.each(updates.deletions,function(title) {
-					if(syncSystemFromServer || !self.wiki.isSystemTiddler(title)) {
-						delete self.tiddlerInfo[title];
-						self.logger.log("Deleting tiddler missing from server:",title);
-						self.wiki.deleteTiddler(title);
-					}
-				});
-				if(updates.modifications.length > 0 || updates.deletions.length > 0) {
-					self.processTaskQueue();
-				}				
-			}
-		});
-	} else if(this.syncadaptor && this.syncadaptor.getSkinnyTiddlers) {
-		this.logger.log("Retrieving skinny tiddler list");
-		cancelNextSync();
-		this.syncadaptor.getSkinnyTiddlers(function(err,tiddlers) {
-			triggerNextSync();
-			// Check for errors
-			if(err) {
-				self.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
-				return;
-			}
-			// Keep track of which tiddlers we already know about have been reported this time
-			var previousTitles = Object.keys(self.tiddlerInfo);
-			// Process each incoming tiddler
-			for(var t=0; t<tiddlers.length; t++) {
-				// Get the incoming tiddler fields, and the existing tiddler
-				var tiddlerFields = tiddlers[t],
-					incomingRevision = tiddlerFields.revision + "",
-					tiddler = self.wiki.tiddlerExists(tiddlerFields.title) && self.wiki.getTiddler(tiddlerFields.title),
-					tiddlerInfo = self.tiddlerInfo[tiddlerFields.title],
-					currRevision = tiddlerInfo ? tiddlerInfo.revision : null,
-					indexInPreviousTitles = previousTitles.indexOf(tiddlerFields.title);
-				if(indexInPreviousTitles !== -1) {
-					previousTitles.splice(indexInPreviousTitles,1);
-				}
-				// Ignore the incoming tiddler if it's the same as the revision we've already got
-				if(currRevision !== incomingRevision) {
-					// Only load the skinny version if we don't already have a fat version of the tiddler
-					if(!tiddler || tiddler.fields.text === undefined) {
-						self.storeTiddler(tiddlerFields);
-					}
-					// Do a full load of this tiddler
-					self.titlesToBeLoaded[tiddlerFields.title] = true;
-				}
-			}
-			// Delete any tiddlers that were previously reported but missing this time
-			$tw.utils.each(previousTitles,function(title) {
-				if(syncSystemFromServer || !self.wiki.isSystemTiddler(title)) {
-					delete self.tiddlerInfo[title];
-					self.logger.log("Deleting tiddler missing from server:",title);
-					self.wiki.deleteTiddler(title);
-				}
-			});
-			self.processTaskQueue();
-		});
-	}
+	this.forceSyncFromServer = true;
+	this.processTaskQueue();
 };
 
 /*
@@ -498,6 +425,7 @@ Syncer.prototype.processTaskQueue = function() {
 	if((!this.syncadaptor.isReady || this.syncadaptor.isReady()) && this.numTasksInProgress === 0) {
 		// Choose the next task to perform
 		var task = this.chooseNextTask();
+		self.logger.log("Chosen next task " + task);
 		// Perform the task if we had one
 		if(typeof task === "object" && task !== null) {
 			this.numTasksInProgress += 1;
@@ -510,7 +438,7 @@ Syncer.prototype.processTaskQueue = function() {
 				} else {
 					self.updateDirtyStatus();
 					// Process the next task
-					self.processTaskQueue.call(self);					
+					self.processTaskQueue.call(self);
 				}
 			});
 		} else {
@@ -518,31 +446,39 @@ Syncer.prototype.processTaskQueue = function() {
 			this.updateDirtyStatus();
 			// And trigger a timeout if there is a pending task
 			if(task === true) {
-				this.triggerTimeout();				
+				this.triggerTimeout(this.taskTimerInterval);
+			} else {
+				this.triggerTimeout(this.pollTimerInterval);
 			}
 		}
 	} else {
-		this.updateDirtyStatus();		
+		this.updateDirtyStatus();
+		this.triggerTimeout(this.taskTimerInterval);
 	}
 };
 
 Syncer.prototype.triggerTimeout = function(interval) {
 	var self = this;
-	if(!this.taskTimerId) {
-		this.taskTimerId = setTimeout(function() {
-			self.taskTimerId = null;
-			self.processTaskQueue.call(self);
-		},interval || self.taskTimerInterval);
+	if(this.taskTimerId) {
+		clearTimeout(this.taskTimerId);
 	}
+	this.taskTimerId = setTimeout(function() {
+		self.taskTimerId = null;
+		self.processTaskQueue.call(self);
+	},interval || self.taskTimerInterval);
 };
 
 /*
-Choose the next sync task. We prioritise saves, then deletes, then loads from the server
+Choose the next sync task. We prioritise saves to the server, then getting updates from the server, then deletes to the server, then loads from the server
 
-Returns either a task object, null if there's no upcoming tasks, or the boolean true if there are pending tasks that aren't yet due
+Returns either:
+* a task object
+* the boolean true if there are pending sync tasks that aren't yet due
+* null if there's no pending sync tasks (just the next poll)
 */
 Syncer.prototype.chooseNextTask = function() {
-	var thresholdLastSaved = (new Date()) - this.throttleInterval,
+	var now = new Date(),
+		thresholdLastSaved = now - this.throttleInterval,
 		havePending = null;
 	// First we look for tiddlers that have been modified locally and need saving back to the server
 	var titles = this.getSyncedTiddlers();
@@ -556,14 +492,18 @@ Syncer.prototype.chooseNextTask = function() {
 				isReadyToSave = !tiddlerInfo || !tiddlerInfo.timestampLastSaved || tiddlerInfo.timestampLastSaved < thresholdLastSaved;
 			if(hasChanged) {
 				if(isReadyToSave) {
-					return new SaveTiddlerTask(this,title); 					
+					return new SaveTiddlerTask(this,title);
 				} else {
 					havePending = true;
 				}
 			}
 		}
 	}
-	// Second, we check tiddlers that are known from the server but not currently in the store, and so need deleting on the server
+	// Second we check for an outstanding sync from server
+	if(this.forceSyncFromServer || (this.timestampLastSyncFromServer && (now.valueOf() >= (this.timestampLastSyncFromServer.valueOf() + this.pollTimerInterval)))) {
+		return new SyncFromServerTask(this);
+	}
+	// Third, we check tiddlers that are known from the server but not currently in the store, and so need deleting on the server
 	titles = Object.keys(this.tiddlerInfo);
 	for(index=0; index<titles.length; index++) {
 		title = titles[index];
@@ -573,13 +513,13 @@ Syncer.prototype.chooseNextTask = function() {
 			return new DeleteTiddlerTask(this,title);
 		}
 	}
-	// Check for tiddlers that need loading
+	// Finally, check for tiddlers that need loading
 	title = Object.keys(this.titlesToBeLoaded)[0];
 	if(title) {
 		delete this.titlesToBeLoaded[title];
 		return new LoadTiddlerTask(this,title);
 	}
-	// No tasks are ready
+	// No tasks are ready now, but might be in the future
 	return havePending;
 };
 
@@ -589,6 +529,10 @@ function SaveTiddlerTask(syncer,title) {
 	this.type = "save";
 }
 
+SaveTiddlerTask.prototype.toString = function() {
+	return "SAVE " + this.title;
+}
+
 SaveTiddlerTask.prototype.run = function(callback) {
 	var self = this,
 		changeCount = this.syncer.wiki.getChangeCount(this.title),
@@ -607,6 +551,7 @@ SaveTiddlerTask.prototype.run = function(callback) {
 				revision: revision,
 				timestampLastSaved: new Date()
 			};
+			self.syncer.logger.log("Updating tiddler info in SaveTiddlerTask.run for " + self.title + " " + JSON.stringify(self.syncer.tiddlerInfo[self.title]));
 			// Invoke the callback
 			callback(null);
 		},{
@@ -624,6 +569,10 @@ function DeleteTiddlerTask(syncer,title) {
 	this.type = "delete";
 }
 
+DeleteTiddlerTask.prototype.toString = function() {
+	return "DELETE " + this.title;
+}
+
 DeleteTiddlerTask.prototype.run = function(callback) {
 	var self = this;
 	this.syncer.logger.log("Dispatching 'delete' task:",this.title);
@@ -633,6 +582,7 @@ DeleteTiddlerTask.prototype.run = function(callback) {
 			return callback(err);
 		}
 		// Remove the info stored about this tiddler
+		self.syncer.logger.log("Deleting tiddler info in DeleteTiddlerTask.run for " + self.title);
 		delete self.syncer.tiddlerInfo[self.title];
 		// Invoke the callback
 		callback(null);
@@ -647,6 +597,10 @@ function LoadTiddlerTask(syncer,title) {
 	this.type = "load";
 }
 
+LoadTiddlerTask.prototype.toString = function() {
+	return "LOAD " + this.title;
+}
+
 LoadTiddlerTask.prototype.run = function(callback) {
 	var self = this;
 	this.syncer.logger.log("Dispatching 'load' task:",this.title);
@@ -664,6 +618,94 @@ LoadTiddlerTask.prototype.run = function(callback) {
 	});
 };
 
+function SyncFromServerTask(syncer) {
+	this.syncer = syncer;
+	this.type = "syncfromserver";
+}
+
+SyncFromServerTask.prototype.toString = function() {
+	return "SYNCFROMSERVER";
+}
+
+SyncFromServerTask.prototype.run = function(callback) {
+	var self = this;
+	var syncSystemFromServer = (self.syncer.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes" ? true : false);
+	var successCallback = function() {
+		self.syncer.forceSyncFromServer = false;
+		self.syncer.timestampLastSyncFromServer = new Date();
+		callback(null);
+	};
+	if(this.syncer.syncadaptor.getUpdatedTiddlers) {
+		self.syncer.logger.log("Retrieving updated tiddler list");
+		this.syncer.syncadaptor.getUpdatedTiddlers(self,function(err,updates) {
+			if(err) {
+				self.syncer.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
+				return callback(err);
+			}
+			if(updates) {
+				$tw.utils.each(updates.modifications,function(title) {
+					self.syncer.titlesToBeLoaded[title] = true;
+				});
+				$tw.utils.each(updates.deletions,function(title) {
+					if(syncSystemFromServer || !self.syncer.wiki.isSystemTiddler(title)) {
+						delete self.syncer.tiddlerInfo[title];
+						self.syncer.logger.log("Deleting tiddler missing from server:",title);
+						self.syncer.wiki.deleteTiddler(title);
+					}
+				});
+			}
+			return successCallback();
+		});
+	} else if(this.syncer.syncadaptor.getSkinnyTiddlers) {
+		this.syncer.logger.log("Retrieving skinny tiddler list");
+		this.syncer.syncadaptor.getSkinnyTiddlers(function(err,tiddlers) {
+			self.syncer.logger.log("Retrieved skinny tiddler list");
+			// Check for errors
+			if(err) {
+				self.syncer.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
+				return callback(err);
+			}
+			// Keep track of which tiddlers we already know about have been reported this time
+			var previousTitles = Object.keys(self.syncer.tiddlerInfo);
+			// Process each incoming tiddler
+			for(var t=0; t<tiddlers.length; t++) {
+				// Get the incoming tiddler fields, and the existing tiddler
+				var tiddlerFields = tiddlers[t],
+					incomingRevision = tiddlerFields.revision + "",
+					tiddler = self.syncer.wiki.tiddlerExists(tiddlerFields.title) && self.syncer.wiki.getTiddler(tiddlerFields.title),
+					tiddlerInfo = self.syncer.tiddlerInfo[tiddlerFields.title],
+					currRevision = tiddlerInfo ? tiddlerInfo.revision : null,
+					indexInPreviousTitles = previousTitles.indexOf(tiddlerFields.title);
+				if(indexInPreviousTitles !== -1) {
+					previousTitles.splice(indexInPreviousTitles,1);
+				}
+				// Ignore the incoming tiddler if it's the same as the revision we've already got
+				if(currRevision !== incomingRevision) {
+					// Only load the skinny version if we don't already have a fat version of the tiddler
+					if(!tiddler || tiddler.fields.text === undefined) {
+						self.syncer.storeTiddler(tiddlerFields);
+					}
+					// Do a full load of this tiddler
+					self.syncer.titlesToBeLoaded[tiddlerFields.title] = true;
+				}
+			}
+			// Delete any tiddlers that were previously reported but missing this time
+			$tw.utils.each(previousTitles,function(title) {
+				if(syncSystemFromServer || !self.syncer.wiki.isSystemTiddler(title)) {
+					delete self.syncer.tiddlerInfo[title];
+					self.syncer.logger.log("Deleting tiddler missing from server:",title);
+					self.syncer.wiki.deleteTiddler(title);
+				}
+			});
+			self.syncer.forceSyncFromServer = false;
+			self.syncer.timestampLastSyncFromServer = new Date();
+			return successCallback();
+		});
+	} else {
+		return successCallback();
+	}
+};
+
 exports.Syncer = Syncer;
 
 })();

core/modules/widgets/list.js

@@ -28,6 +28,18 @@ Inherit from the base widget class
 */
 ListWidget.prototype = new Widget();
 
+ListWidget.prototype.initialise = function(parseTreeNode,options) {
+	// Bail if parseTreeNode is undefined, meaning that the ListWidget constructor was called without any arguments so that it can be subclassed
+	if(parseTreeNode === undefined) {
+		return;
+	}
+	// First call parent constructor to set everything else up
+	Widget.prototype.initialise.call(this,parseTreeNode,options);
+	// Now look for <$list-template> and <$list-empty> widgets as immediate child widgets
+	// This is safe to do during initialization because parse trees never change after creation
+	this.findExplicitTemplates();
+}
+
 /*
 Render this widget into the DOM
 */
@@ -68,8 +80,6 @@ ListWidget.prototype.execute = function() {
 	this.counterName = this.getAttribute("counter");
 	this.storyViewName = this.getAttribute("storyview");
 	this.historyTitle = this.getAttribute("history");
-	// Look for <$list-template> and <$list-empty> widgets as immediate child widgets
-	this.findExplicitTemplates();
 	// Compose the list elements
 	this.list = this.getTiddlerList();
 	var members = [],
@@ -92,6 +102,7 @@ ListWidget.prototype.findExplicitTemplates = function() {
 	var self = this;
 	this.explicitListTemplate = null;
 	this.explicitEmptyTemplate = null;
+	this.hasTemplateInBody = false;
 	var searchChildren = function(childNodes) {
 		$tw.utils.each(childNodes,function(node) {
 			if(node.type === "list-template") {
@@ -100,6 +111,8 @@ ListWidget.prototype.findExplicitTemplates = function() {
 				self.explicitEmptyTemplate = node.children;
 			} else if(node.type === "element" && node.tag === "p") {
 				searchChildren(node.children);
+			} else {
+				self.hasTemplateInBody = true;
 			}
 		});
 	};
@@ -160,11 +173,11 @@ ListWidget.prototype.makeItemTemplate = function(title,index) {
 			// Check for a <$list-item> widget
 			if(this.explicitListTemplate) {
 				templateTree = this.explicitListTemplate;
-			} else if (!this.explicitEmptyTemplate) {
+			} else if(this.hasTemplateInBody) {
 				templateTree = this.parseTreeNode.children;
 			}
 		}
-		if(!templateTree) {
+		if(!templateTree || templateTree.length === 0) {
 			// Default template is a link to the title
 			templateTree = [{type: "element", tag: this.parseTreeNode.isBlock ? "div" : "span", children: [{type: "link", attributes: {to: {type: "string", value: title}}, children: [
 				{type: "text", text: title}
@@ -414,4 +427,27 @@ ListItemWidget.prototype.refresh = function(changedTiddlers) {
 
 exports.listitem = ListItemWidget;
 
+/*
+Make <$list-template> and <$list-empty> widgets that do nothing
+*/
+var ListTemplateWidget = function(parseTreeNode,options) {
+	// Main initialisation inherited from widget.js
+	this.initialise(parseTreeNode,options);
+};
+ListTemplateWidget.prototype = new Widget();
+ListTemplateWidget.prototype.render = function() {}
+ListTemplateWidget.prototype.refresh = function() { return false; }
+
+exports["list-template"] = ListTemplateWidget;
+
+var ListEmptyWidget = function(parseTreeNode,options) {
+	// Main initialisation inherited from widget.js
+	this.initialise(parseTreeNode,options);
+};
+ListEmptyWidget.prototype = new Widget();
+ListEmptyWidget.prototype.render = function() {}
+ListEmptyWidget.prototype.refresh = function() { return false; }
+
+exports["list-empty"] = ListEmptyWidget;
+
 })();