Add server sent events (#5279)

* Create server-sent-events.js

* Create sse-change-listener.js

* Implement server sent events

* Convert to ES5 and wrap in function

* Use the host string from tiddlyweb

* Improve comments in sse-server.js

* Can't use object reference as key

* Add retry timeout

* Fix a bug

* bug fix

* Fix formatting

* Fix ES5 compat

* capitalize comments

* more fixes

* Refactor tiddlywek/sse-server.js

* Extract helper functions for handling wikis and connections.
* Replace JSDoc comments.
* Fix formatting according to TW core.
* Simplify the logic for adding and removing connections.

* Fix formatting of tiddlyweb/sse-client.js

Fix formatting according to TW core.

* Fix formatting of server-sent-events.js

Fix formatting and comments following TW core guidelines.

* Extract a debounce function in sse-client.js

* Avoid using startsWith in server-sent-events.js

startsWith is part of ES2015, while TiddlyWiki uses the 5.1 dialect.

* New sse-enabled WebServer parameter

* If not set to "yes", disabled SSE request handling.
* Add documentation for the parameter in core/language/en-GB/Help/listen.tid
* Add new tiddler editions/tw5.com/tiddlers/webserver/WebServer Parameter_ sse-enabled.tid

* Disable polling for changes if SSE is enabled

* Add sse_enabled to /status JSON response
* Store syncer polling status in $:/config/SyncDisablePolling
* Handled disabling polling in core/modules/syncer.js

* Simply boolean logic in syncer.js

* Delete trailing whitespaces in syncer.js

Co-authored-by: Arlen22 <arlenbee@gmail.com>

core/language/en-GB/Help/listen.tid

@@ -22,6 +22,7 @@ All parameters are optional with safe defaults, and can be specified in any orde
 * ''readers'' - comma separated list of principals allowed to read from this wiki
 * ''writers'' - comma separated list of principals allowed to write to this wiki
 * ''csrf-disable'' - set to "yes" to disable CSRF checks (defaults to "no")
+* ''sse-enabled'' - set to "yes" to enable Server-sent events (defaults to "no")
 * ''root-tiddler'' - the tiddler to serve at the root (defaults to "$:/core/save/all")
 * ''root-render-type'' - the content type to which the root tiddler should be rendered (defaults to "text/plain")
 * ''root-serve-type'' - the content type with which the root tiddler should be served (defaults to "text/html")

core/modules/server/routes/get-status.js

@@ -22,6 +22,7 @@ exports.handler = function(request,response,state) {
 		username: state.authenticatedUsername || state.server.get("anon-username") || "",
 		anonymous: !state.authenticatedUsername,
 		read_only: !state.server.isAuthorized("writers",state.authenticatedUsername),
+		sse_enabled: state.server.get("sse-enabled") === "yes",
 		space: {
 			recipe: "default"
 		},

core/modules/server/server-sent-events.js

@@ -0,0 +1,70 @@
+/*\
+title: $:/core/modules/server/server-sent-events.js
+type: application/javascript
+module-type: library
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+parameters:
+		prefix - usually the plugin path, such as `plugins/tiddlywiki/tiddlyweb`. The
+			route will match `/events/${prefix}` exactly.
+
+		handler - a function that will be called each time a request comes in with the
+			request and state from the route and an emit function to call.
+*/
+
+var ServerSentEvents = function ServerSentEvents(prefix, handler) {
+	this.handler = handler;
+	this.prefix = prefix;
+};
+
+ServerSentEvents.prototype.getExports = function() {
+	return {
+		bodyFormat: "stream",
+		method: "GET",
+		path: new RegExp("^/events/" + this.prefix + "$"),
+		handler: this.handleEventRequest.bind(this)
+	};
+};
+
+ServerSentEvents.prototype.handleEventRequest = function(request,response,state) {
+	if(ServerSentEvents.prototype.isEventStreamRequest(request)) {
+		response.writeHead(200, {
+			"Content-Type": "text/event-stream",
+			"Cache-Control": "no-cache",
+			"Connection": "keep-alive"
+		});
+		this.handler(request,state,this.emit.bind(this,response),this.end.bind(this,response));
+	} else {
+		response.writeHead(406,"Not Acceptable",{});
+		response.end();
+	}
+};
+
+ServerSentEvents.prototype.isEventStreamRequest = function(request) {
+	return request.headers.accept &&
+		request.headers.accept.match(/^text\/event-stream/);
+};
+
+ServerSentEvents.prototype.emit = function(response,event,data) {
+	if(typeof event !== "string" || event.indexOf("\n") !== -1) {
+		throw new Error("Type must be a single-line string");
+	}
+	if(typeof data !== "string" || data.indexOf("\n") !== -1) {
+		throw new Error("Data must be a single-line string");
+	}
+	response.write("event: " + event + "\ndata: " + data + "\n\n", "utf8");
+};
+
+ServerSentEvents.prototype.end = function(response) {
+	response.end();
+};
+
+exports.ServerSentEvents = ServerSentEvents;
+
+})();

core/modules/syncer.js

@@ -20,6 +20,7 @@ Syncer.prototype.titleIsAnonymous = "$:/status/IsAnonymous";
 Syncer.prototype.titleIsReadOnly = "$:/status/IsReadOnly";
 Syncer.prototype.titleUserName = "$:/status/UserName";
 Syncer.prototype.titleSyncFilter = "$:/config/SyncFilter";
+Syncer.prototype.titleSyncDisablePolling = "$:/config/SyncDisablePolling";
 Syncer.prototype.titleSyncPollingInterval = "$:/config/SyncPollingInterval";
 Syncer.prototype.titleSyncDisableLazyLoading = "$:/config/SyncDisableLazyLoading";
 Syncer.prototype.titleSavedNotification = "$:/language/Notifications/Save/Done";
@@ -89,7 +90,7 @@ function Syncer(options) {
 		if(filteredChanges.length > 0) {
 			self.processTaskQueue();
 		} else {
-			// Look for deletions of tiddlers we're already syncing	
+			// Look for deletions of tiddlers we're already syncing
 			var outstandingDeletion = false
 			$tw.utils.each(changes,function(change,title,object) {
 				if(change.deleted && $tw.utils.hop(self.tiddlerInfo,title)) {
@@ -121,7 +122,7 @@ function Syncer(options) {
 				self.login(username,password,function() {});
 			} else {
 				// No username and password, so we display a prompt
-				self.handleLoginEvent();				
+				self.handleLoginEvent();
 			}
 		});
 		$tw.rootWidget.addEventListener("tm-logout",function() {
@@ -138,7 +139,7 @@ function Syncer(options) {
 	if(!this.disableUI && this.wiki.getTiddlerText(this.titleSyncDisableLazyLoading) !== "yes") {
 		this.wiki.addEventListener("lazyLoad",function(title) {
 			self.handleLazyLoadEvent(title);
-		});		
+		});
 	}
 	// Get the login status
 	this.getStatus(function(err,isLoggedIn) {
@@ -173,8 +174,8 @@ Syncer.prototype.getTiddlerRevision = function(title) {
 	if(this.syncadaptor && this.syncadaptor.getTiddlerRevision) {
 		return this.syncadaptor.getTiddlerRevision(title);
 	} else {
-		return this.wiki.getTiddler(title).fields.revision;	
+		return this.wiki.getTiddler(title).fields.revision;
-	} 
+	}
 };
 
 /*
@@ -267,7 +268,7 @@ Syncer.prototype.getStatus = function(callback) {
 		// Mark us as not logged in
 		this.wiki.addTiddler({title: this.titleIsLoggedIn,text: "no"});
 		// Get login status
-		this.syncadaptor.getStatus(function(err,isLoggedIn,username,isReadOnly,isAnonymous) {
+		this.syncadaptor.getStatus(function(err,isLoggedIn,username,isReadOnly,isAnonymous,isPollingDisabled) {
 			if(err) {
 				self.logger.alert(err);
 			} else {
@@ -278,6 +279,9 @@ Syncer.prototype.getStatus = function(callback) {
 				if(isLoggedIn) {
 					self.wiki.addTiddler({title: self.titleUserName,text: username || ""});
 				}
+				if(isPollingDisabled) {
+					self.wiki.addTiddler({title: self.titleSyncDisablePolling, text: "yes"});
+				}
 			}
 			// Invoke the callback
 			if(callback) {
@@ -301,12 +305,15 @@ Syncer.prototype.syncFromServer = function() {
 			}
 		},
 		triggerNextSync = function() {
-			self.pollTimerId = setTimeout(function() {
+			if(pollingEnabled) {
-				self.pollTimerId = null;
+				self.pollTimerId = setTimeout(function() {
-				self.syncFromServer.call(self);
+					self.pollTimerId = null;
-			},self.pollTimerInterval);
+					self.syncFromServer.call(self);
+				},self.pollTimerInterval);
+			}
 		},
-		syncSystemFromServer = (self.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes" ? true : false);
+		syncSystemFromServer = (self.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes"),
+		pollingEnabled = (self.wiki.getTiddlerText(self.titleSyncDisablePolling) !== "yes");
 	if(this.syncadaptor && this.syncadaptor.getUpdatedTiddlers) {
 		this.logger.log("Retrieving updated tiddler list");
 		cancelNextSync();
@@ -329,7 +336,7 @@ Syncer.prototype.syncFromServer = function() {
 				});
 				if(updates.modifications.length > 0 || updates.deletions.length > 0) {
 					self.processTaskQueue();
-				}				
+				}
 			}
 		});
 	} else if(this.syncadaptor && this.syncadaptor.getSkinnyTiddlers) {
@@ -509,7 +516,7 @@ Syncer.prototype.processTaskQueue = function() {
 				} else {
 					self.updateDirtyStatus();
 					// Process the next task
-					self.processTaskQueue.call(self);					
+					self.processTaskQueue.call(self);
 				}
 			});
 		} else {
@@ -517,11 +524,11 @@ Syncer.prototype.processTaskQueue = function() {
 			this.updateDirtyStatus();
 			// And trigger a timeout if there is a pending task
 			if(task === true) {
-				this.triggerTimeout();				
+				this.triggerTimeout();
 			}
 		}
 	} else {
-		this.updateDirtyStatus();		
+		this.updateDirtyStatus();
 	}
 };
 
@@ -555,7 +562,7 @@ Syncer.prototype.chooseNextTask = function() {
 				isReadyToSave = !tiddlerInfo || !tiddlerInfo.timestampLastSaved || tiddlerInfo.timestampLastSaved < thresholdLastSaved;
 			if(hasChanged) {
 				if(isReadyToSave) {
-					return new SaveTiddlerTask(this,title); 					
+					return new SaveTiddlerTask(this,title);
 				} else {
 					havePending = true;
 				}

editions/tw5.com/tiddlers/webserver/WebServer Parameter_ sse-enabled.tid

@@ -0,0 +1,11 @@
+caption: sse-enabled
+created: 20210113204602693
+modified: 20210113205535065
+tags: [[WebServer Parameters]]
+title: WebServer Parameter: sse-enabled
+type: text/vnd.tiddlywiki
+
+The [[web server configuration parameter|WebServer Parameters]] ''sse-enabled'' enabled [[Server sent events|https://en.wikipedia.org/wiki/Server-sent_events]], allowing changes to be propagated in almost real time to all browser windows or tabs.
+
+Setting ''sse-enabled'' to `yes` enables Server-sent events; `no`, or any other value, disables them.
+

plugins/tiddlywiki/tiddlyweb/sse-client.js

@@ -0,0 +1,53 @@
+/*\
+title: $:/plugins/tiddlywiki/tiddlyweb/sse-client.js
+type: application/javascript
+module-type: startup
+
+GET /recipes/default/tiddlers/:title
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.name = "/events/plugins/tiddlywiki/tiddlyweb";
+exports.after = ["startup"];
+exports.synchronous = true;
+exports.platforms = ["browser"];
+exports.startup = function() {
+	// Make sure we're actually being used
+	if($tw.syncadaptor.name !== "tiddlyweb") {
+		return;
+	}
+	// Get the mount point in case a path prefix is used
+	var host = $tw.syncadaptor.getHost();
+	// Make sure it ends with a slash (it usually does)
+	if(host[host.length - 1] !== "/") {
+		host += "/";
+	}
+	// Setup the event listener
+	setupEvents(host);
+};
+
+function debounce(callback) {
+	var timeout = null;
+	return function() {
+		clearTimeout(timeout);
+		timeout = setTimeout(callback,$tw.syncer.throttleInterval);
+	};
+}
+
+function setupEvents(host) {
+	var events = new EventSource(host + "events/plugins/tiddlywiki/tiddlyweb");
+	var debouncedSync = debounce($tw.syncer.syncFromServer.bind($tw.syncer));
+	events.addEventListener("change",debouncedSync);
+	events.onerror = function() {
+		events.close();
+		setTimeout(function() {
+			setupEvents(host);
+		},$tw.syncer.errorRetryInterval);
+	};
+}
+})();

plugins/tiddlywiki/tiddlyweb/sse-server.js

@@ -0,0 +1,94 @@
+/*\
+title: $:/plugins/tiddlywiki/tiddlyweb/sse-server.js
+type: application/javascript
+module-type: route
+
+GET /events/plugins/tiddlywiki/tiddlyweb
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+var wikis = [];
+var connections = [];
+
+/*
+Setup up the array for this wiki and add the change listener
+*/
+function setupWiki(wiki) {
+	var index = wikis.length;
+	// Add a new array for this wiki (object references work as keys)
+	wikis.push(wiki);
+	connections.push([]);
+	// Listen to change events for this wiki
+	wiki.addEventListener("change",function(changes) {
+		var jsonChanges = JSON.stringify(changes);
+		getWikiConnections(wiki).forEach(function(item) {
+			item.emit("change",jsonChanges);
+		});
+	});
+	return index;
+}
+
+/*
+Setup this particular wiki if we haven't seen it before
+*/
+function ensureWikiSetup(wiki) {
+	if(wikis.indexOf(wiki) === -1) {
+		setupWiki(wiki);
+	}
+}
+
+/*
+Return the array of connections for a particular wiki
+*/
+function getWikiConnections(wiki) {
+	return connections[wikis.indexOf(wiki)];
+}
+
+function addWikiConnection(wiki,connection) {
+	getWikiConnections(wiki).push(connection);
+}
+
+function removeWikiConnection(wiki,connection) {
+	var wikiConnections = getWikiConnections(wiki);
+	var index = wikiConnections.indexOf(connection);
+	if(index !== -1) {
+		wikiConnections.splice(index,1);
+	}
+}
+
+function handleConnection(request,state,emit,end) {
+	if(isDisabled(state)) {
+		return;
+	}
+
+	ensureWikiSetup(state.wiki);
+	// Add the connection to the list of connections for this wiki
+	var connection = {
+		request: request,
+		state: state,
+		emit: emit,
+		end: end
+	};
+	addWikiConnection(state.wiki,connection);
+	request.on("close",function() {
+		removeWikiConnection(state.wiki,connection);
+	});
+}
+
+function isDisabled(state) {
+	return state.server.get("sse-enabled") !== "yes";
+}
+
+// Import the ServerSentEvents class
+var ServerSentEvents = require("$:/core/modules/server/server-sent-events.js").ServerSentEvents;
+// Instantiate the class
+var events = new ServerSentEvents("plugins/tiddlywiki/tiddlyweb", handleConnection);
+// Export the route definition for this server sent events instance
+module.exports = events.getExports();
+
+})();

plugins/tiddlywiki/tiddlyweb/tiddlywebadaptor.js

@@ -91,10 +91,12 @@ TiddlyWebAdaptor.prototype.getStatus = function(callback) {
 				self.isLoggedIn = json.username !== "GUEST";
 				self.isReadOnly = !!json["read_only"];
 				self.isAnonymous = !!json.anonymous;
+
+				var isSseEnabled = !!json.sse_enabled;
 			}
 			// Invoke the callback if present
 			if(callback) {
-				callback(null,self.isLoggedIn,json.username,self.isReadOnly,self.isAnonymous);
+				callback(null,self.isLoggedIn,json.username,self.isReadOnly,self.isAnonymous,isSseEnabled);
 			}
 		}
 	});