Merge remote-tracking branch 'upstream/master' into feat/section-mark

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ on:
       - master
       - tiddlywiki-com
 env:
-  NODE_VERSION: "12"
+  NODE_VERSION: "18"
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -14,7 +14,13 @@ jobs:
       - uses: actions/setup-node@v1
         with:
           node-version: "${{ env.NODE_VERSION }}"
-      - run: "./bin/test.sh"
+      - run: "./bin/ci-test.sh"
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
   build-prerelease:
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/master'
@@ -54,6 +60,7 @@ jobs:
       TW5_BUILD_TIDDLYWIKI: "./node_modules/tiddlywiki/tiddlywiki.js"
       TW5_BUILD_MAIN_EDITION: "./editions/tw5.com"
       TW5_BUILD_OUTPUT: "./output"
+      TW5_BUILD_ARCHIVE: "./output"
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1

.gitignore

@@ -5,4 +5,7 @@
 tmp/
 output/
 node_modules/
-
+/test-results/
+/playwright-report/
+/playwright/.cache/
+$__StoryList.tid

bin/build-site.sh

@@ -5,7 +5,7 @@
 # Default to the current version number for building the plugin library
 
 if [  -z "$TW5_BUILD_VERSION" ]; then
-    TW5_BUILD_VERSION=v5.3.2
+    TW5_BUILD_VERSION=v5.3.3
 fi
 
 echo "Using TW5_BUILD_VERSION as [$TW5_BUILD_VERSION]"
@@ -84,10 +84,27 @@ echo -e -n "title: $:/build\ncommit: $TW5_BUILD_COMMIT\n\n$TW5_BUILD_DETAILS\n"
 
 ######################################################
 #
-# Core distribution
+# Core distributions
 #
 ######################################################
 
+# Conditionally build archive if $TW5_BUILD_ARCHIVE variable is set, otherwise do nothing
+#
+# /archive/Empty-TiddlyWiki-<version>.html	Empty archived version
+# /archive/TiddlyWiki-<version>.html		Full archived version
+
+if [ -n "$TW5_BUILD_ARCHIVE" ]; then
+
+node $TW5_BUILD_TIDDLYWIKI \
+	$TW5_BUILD_MAIN_EDITION \
+	--verbose \
+	--version \
+	--load $TW5_BUILD_OUTPUT/build.tid \
+	--output $TW5_BUILD_ARCHIVE \
+	--build archive \
+	|| exit 1
+fi
+
 # /index.html			Main site
 # /favicon.ico			Favicon for main site
 # /static.html			Static rendering of default tiddlers
@@ -95,6 +112,7 @@ echo -e -n "title: $:/build\ncommit: $TW5_BUILD_COMMIT\n\n$TW5_BUILD_DETAILS\n"
 # /static/*				Static single tiddlers
 # /static/static.css	Static stylesheet
 # /static/favicon.ico	Favicon for static pages
+
 node $TW5_BUILD_TIDDLYWIKI \
 	$TW5_BUILD_MAIN_EDITION \
 	--verbose \
@@ -138,6 +156,28 @@ node $TW5_BUILD_TIDDLYWIKI \
 	--build index favicon static \
 	|| exit 1
 
+# /tour.html			tour edition
+node $TW5_BUILD_TIDDLYWIKI \
+	./editions/tour \
+	--verbose \
+	--output $TW5_BUILD_OUTPUT \
+	--rendertiddler $:/core/save/all tour.html text/plain \
+	|| exit 1
+
+# /dev/index.html			Developer docs
+# /dev/favicon.ico			Favicon for dev site
+# /dev/static.html			Static rendering of default tiddlers
+# /dev/alltiddlers.html		Static rendering of all tiddlers
+# /dev/static/*				Static single tiddlers
+# /dev/static/static.css	Static stylesheet
+node $TW5_BUILD_TIDDLYWIKI \
+	./editions/dev \
+	--verbose \
+	--load $TW5_BUILD_OUTPUT/build.tid \
+	--output $TW5_BUILD_OUTPUT/dev \
+	--build index favicon static \
+	|| exit 1
+
 # /share.html				Custom edition for sharing via the URL
 node $TW5_BUILD_TIDDLYWIKI \
 	./editions/share \
@@ -353,6 +393,17 @@ node $TW5_BUILD_TIDDLYWIKI \
 	--rendertiddler $:/core/save/empty plugins/tiddlywiki/highlight/empty.html text/plain \
 	|| exit 1
 
+# /plugins/tiddlywiki/geospatial/index.html		Demo wiki with geospatial plugin
+# /plugins/tiddlywiki/geospatial/empty.html		Empty wiki with geospatial plugin
+node $TW5_BUILD_TIDDLYWIKI \
+	./editions/geospatialdemo \
+	--verbose \
+	--load $TW5_BUILD_OUTPUT/build.tid \
+	--output $TW5_BUILD_OUTPUT \
+	--rendertiddler $:/core/save/all plugins/tiddlywiki/geospatial/index.html text/plain \
+	--rendertiddler $:/core/save/empty plugins/tiddlywiki/geospatial/empty.html text/plain \
+	|| exit 1
+
 ######################################################
 #
 # Language editions

bin/ci-test.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# test TiddlyWiki5 for tiddlywiki.com
+
+node ./tiddlywiki.js \
+	./editions/test \
+	--verbose \
+	--version \
+	--rendertiddler $:/core/save/all test.html text/plain \
+	--test \
+	|| exit 1
+
+npm install playwright @playwright/test
+npx playwright install chromium firefox --with-deps
+
+npx playwright test

boot/boot.js

@@ -177,6 +177,7 @@ document: defaults to current document
 eventListeners: array of event listeners (this option won't work until $tw.utils.addEventListeners() has been loaded)
 */
 $tw.utils.domMaker = function(tag,options) {
+	var options = options || {};
 	var doc = options.document || document;
 	var element = doc.createElementNS(options.namespace || "http://www.w3.org/1999/xhtml",tag);
 	if(options["class"]) {
@@ -218,9 +219,34 @@ $tw.utils.error = function(err) {
 			heading = dm("h1",{text: errHeading}),
 			prompt = dm("div",{text: promptMsg, "class": "tc-error-prompt"}),
 			message = dm("div",{text: err, "class":"tc-error-message"}),
-			button = dm("div",{children: [dm("button",{text: ( $tw.language == undefined ? "close" : $tw.language.getString("Buttons/Close/Caption") )})], "class": "tc-error-prompt"}),
+			closeButton = dm("div",{children: [dm("button",{text: ( $tw.language == undefined ? "close" : $tw.language.getString("Buttons/Close/Caption") )})], "class": "tc-error-prompt"}),
-			form = dm("form",{children: [heading,prompt,message,button], "class": "tc-error-form"});
+			downloadButton = dm("div",{children: [dm("button",{text: ( $tw.language == undefined ? "download tiddlers" : $tw.language.getString("Buttons/EmergencyDownload/Caption") )})], "class": "tc-error-prompt"}),
+			form = dm("form",{children: [heading,prompt,downloadButton,message,closeButton], "class": "tc-error-form"});
 		document.body.insertBefore(form,document.body.firstChild);
+		downloadButton.addEventListener("click",function(event) {
+			if($tw && $tw.wiki) {
+				var tiddlers = [];
+				$tw.wiki.each(function(tiddler,title) {
+					tiddlers.push(tiddler.fields);
+				});
+				var link = dm("a"),
+					text = JSON.stringify(tiddlers);
+				if(Blob !== undefined) {
+					var blob = new Blob([text], {type: "text/html"});
+					link.setAttribute("href", URL.createObjectURL(blob));
+				} else {
+					link.setAttribute("href","data:text/html," + encodeURIComponent(text));
+				}
+				link.setAttribute("download","emergency-tiddlers-" + (new Date()) + ".json");
+				document.body.appendChild(link);
+				link.click();
+				document.body.removeChild(link);
+			} else {
+				alert("Emergency tiddler download is not available");
+			}
+			event.preventDefault();
+			return false;
+		},true);
 		form.addEventListener("submit",function(event) {
 			document.body.removeChild(form);
 			event.preventDefault();
@@ -786,6 +812,7 @@ $tw.utils.Crypto = function() {
 			}
 			return outputText;
 		};
+	$tw.sjcl = sjcl;
 	this.setPassword = function(newPassword) {
 		currentPassword = newPassword;
 		this.updateCryptoStateTiddler();
@@ -1967,10 +1994,10 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) {
 					var value = tiddler[name];
 					switch(fieldInfo.source) {
 						case "subdirectories":
-							value = path.relative(rootPath, filename).split('/').slice(0, -1);
+							value = path.relative(rootPath, filename).split(path.sep).slice(0, -1);
 							break;
 						case "filepath":
-							value = path.relative(rootPath, filename);
+							value = path.relative(rootPath, filename).split(path.sep).join('/');
 							break;
 						case "filename":
 							value = path.basename(filename);
@@ -2161,6 +2188,8 @@ Returns an array of search paths
 */
 $tw.getLibraryItemSearchPaths = function(libraryPath,envVar) {
 	var pluginPaths = [path.resolve($tw.boot.corePath,libraryPath)],
+		env;
+	if(envVar) {
 		env = process.env[envVar];
 		if(env) {
 			env.split(path.delimiter).map(function(item) {
@@ -2169,6 +2198,7 @@ $tw.getLibraryItemSearchPaths = function(libraryPath,envVar) {
 				}
 			});
 		}
+	}
 	return pluginPaths;
 };
 
@@ -2438,6 +2468,7 @@ $tw.boot.initStartup = function(options) {
 	$tw.utils.registerFileType("image/svg+xml","utf8",".svg",{flags:["image"]});
 	$tw.utils.registerFileType("image/vnd.microsoft.icon","base64",".ico",{flags:["image"]});
 	$tw.utils.registerFileType("image/x-icon","base64",".ico",{flags:["image"]});
+	$tw.utils.registerFileType("application/wasm","base64",".wasm");
 	$tw.utils.registerFileType("application/font-woff","base64",".woff");
 	$tw.utils.registerFileType("application/x-font-ttf","base64",".woff");
 	$tw.utils.registerFileType("application/font-woff2","base64",".woff2");
@@ -2452,8 +2483,12 @@ $tw.boot.initStartup = function(options) {
 	$tw.utils.registerFileType("text/x-markdown","utf8",[".md",".markdown"]);
 	$tw.utils.registerFileType("application/enex+xml","utf8",".enex");
 	$tw.utils.registerFileType("application/vnd.openxmlformats-officedocument.wordprocessingml.document","base64",".docx");
+	$tw.utils.registerFileType("application/msword","base64",".doc");
 	$tw.utils.registerFileType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","base64",".xlsx");
+	$tw.utils.registerFileType("application/excel","base64",".xls");
+	$tw.utils.registerFileType("application/vnd.ms-excel","base64",".xls");
 	$tw.utils.registerFileType("application/vnd.openxmlformats-officedocument.presentationml.presentation","base64",".pptx");
+	$tw.utils.registerFileType("application/mspowerpoint","base64",".ppt");
 	$tw.utils.registerFileType("text/x-bibtex","utf8",".bib",{deserializerType:"application/x-bibtex"});
 	$tw.utils.registerFileType("application/x-bibtex","utf8",".bib");
 	$tw.utils.registerFileType("application/epub+zip","base64",".epub");
@@ -2674,6 +2709,18 @@ $tw.hooks.addHook = function(hookName,definition) {
 	}
 };
 
+/*
+Delete hooks from the hashmap
+*/
+$tw.hooks.removeHook = function(hookName,definition) {
+	if($tw.utils.hop($tw.hooks.names,hookName)) {
+		var p = $tw.hooks.names[hookName].indexOf(definition);
+		if(p !== -1) {
+			$tw.hooks.names[hookName].splice(p, 1);
+		}
+	}
+};
+
 /*
 Invoke the hook by key
 */

boot/sjcl.js.meta

@@ -1,3 +0,0 @@
-title: $:/library/sjcl.js
-type: application/javascript
-library: yes

boot/tiddlywiki.files

@@ -0,0 +1,32 @@
+{
+	"tiddlers": [
+		{
+			"file": "sjcl.js",
+			"fields": {
+				"title": "$:/library/sjcl.js",
+				"type": "application/javascript",
+				"library": "yes"
+			},
+			"prefix": "(function(define) {\n",
+			"suffix": "\n})(function (_,defined){window.sjcl = defined()})\n"
+		},
+		{
+			"file": "boot.js",
+			"fields": {
+				"title": "$:/boot/boot.js",
+				"type": "application/javascript"
+			}
+		},
+		{
+			"file": "bootprefix.js",
+			"fields": {
+				"title": "$:/boot/bootprefix.js",
+				"type": "application/javascript"
+			}
+		},
+		{
+			"file": "boot.css.tid",
+			"isTiddlerFile": true
+		}
+	]
+}

core/copyright.tid

@@ -4,7 +4,7 @@ type: text/plain
 TiddlyWiki created by Jeremy Ruston, (jeremy [at] jermolene [dot] com)
 
 Copyright (c) 2004-2007, Jeremy Ruston
-Copyright (c) 2007-2023, UnaMesa Association
+Copyright (c) 2007-2024, UnaMesa Association
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

core/images/standard-layout.tid

@@ -0,0 +1,7 @@
+title: $:/core/images/standard-layout
+tags: $:/tags/Image
+
+\parameters (size:"22pt")
+<svg width=<<size>> height=<<size>> class="tc-image-standard-layout tc-image-button" viewBox="0 0 128 128">
+    <path d="M71.93 72A8.07 8.07 0 0 1 80 80.07v7.86A8.071 8.071 0 0 1 71.93 96H8.07A8.067 8.067 0 0 1 0 87.93v-7.86A8.072 8.072 0 0 1 8.07 72h63.86Zm0 32a8.07 8.07 0 0 1 8.07 8.07v7.86a8.071 8.071 0 0 1-8.07 8.07H8.07A8.067 8.067 0 0 1 0 119.93v-7.86A8.072 8.072 0 0 1 8.07 104h63.86Zm0-104A8.068 8.068 0 0 1 80 8.07v47.86A8.073 8.073 0 0 1 71.93 64H8.07A8.07 8.07 0 0 1 0 55.93V8.07A8.072 8.072 0 0 1 8.07 0h63.86Zm48 0c2.14 0 4.193.85 5.706 2.364A8.067 8.067 0 0 1 128 8.07v111.86c0 2.14-.85 4.193-2.364 5.706A8.067 8.067 0 0 1 119.93 128H96.07c-2.14 0-4.193-.85-5.706-2.364A8.067 8.067 0 0 1 88 119.93V8.07c0-2.14.85-4.193 2.364-5.706A8.067 8.067 0 0 1 96.07 0h23.86ZM116 24h-16a3.995 3.995 0 0 0-2.828 1.172 3.995 3.995 0 0 0 0 5.656A3.995 3.995 0 0 0 100 32h16a3.995 3.995 0 0 0 2.828-1.172 3.995 3.995 0 0 0 0-5.656A3.995 3.995 0 0 0 116 24Z"/>
+</svg>

core/language/en-GB/Buttons.multids

@@ -28,6 +28,7 @@ Encryption/ClearPassword/Caption: clear password
 Encryption/ClearPassword/Hint: Clear the password and save this wiki without encryption
 Encryption/SetPassword/Caption: set password
 Encryption/SetPassword/Hint: Set a password for saving this wiki with encryption
+EmergencyDownload/Caption: download tiddlers as json
 ExportPage/Caption: export all
 ExportPage/Hint: Export all tiddlers
 ExportTiddler/Caption: export tiddler

core/language/en-GB/ControlPanel.multids

@@ -206,6 +206,12 @@ Stylesheets/Caption: Stylesheets
 Stylesheets/Expand/Caption: Expand All
 Stylesheets/Hint: This is the rendered CSS of the current stylesheet tiddlers tagged with <<tag "$:/tags/Stylesheet">>
 Stylesheets/Restore/Caption: Restore
+TestCases/Caption: Test Cases
+TestCases/Hint: Test cases are self contained examples for testing and learning
+TestCases/All/Caption: All Test Cases
+TestCases/All/Hint: All Test Cases
+TestCases/Failed/Caption: Failed Test Cases
+TestCases/Failed/Hint: Only Failed Test Cases
 Theme/Caption: Theme
 Theme/Prompt: Current theme:
 TiddlerFields/Caption: Tiddler Fields

core/language/en-GB/Docs/ModuleTypes.multids

@@ -9,7 +9,7 @@ config: Data to be inserted into `$tw.config`.
 filteroperator: Individual filter operator methods.
 global: Global data to be inserted into `$tw`.
 info: Publishes system information via the [[$:/temp/info-plugin]] pseudo-plugin.
-isfilteroperator: Operands for the ''is'' filter operator.
+isfilteroperator: Parameters for the ''is'' filter operator.
 library: Generic module type for general purpose JavaScript modules.
 macro: JavaScript macro definitions.
 parser: Parsers for different content types.

core/language/en-GB/Docs/PaletteColours.multids

@@ -65,6 +65,9 @@ sidebar-tab-foreground-selected: Sidebar tab foreground for selected tabs
 sidebar-tab-foreground: Sidebar tab foreground
 sidebar-tiddler-link-foreground-hover: Sidebar tiddler link foreground hover
 sidebar-tiddler-link-foreground: Sidebar tiddler link foreground
+testcase-accent-level-1: Test case accent colour with no nesting
+testcase-accent-level-2: Test case accent colour with 2nd level nesting
+testcase-accent-level-3: Test case accent colour with 3rd level nesting or higher
 site-title-foreground: Site title foreground
 static-alert-foreground: Static alert foreground
 tab-background-selected: Tab background for selected tabs

core/language/en-GB/Exporters.multids

@@ -3,4 +3,4 @@ title: $:/language/Exporters/
 StaticRiver: Static HTML
 JsonFile: JSON file
 CsvFile: CSV file
-TidFile: ".tid" file
+TidFile: TID text file

core/language/en-GB/Fields.multids

@@ -4,6 +4,7 @@ _canonical_uri: The full URI of an external image tiddler
 author: Name of the author of a plugin
 bag: The name of the bag from which a tiddler came
 caption: The text to be displayed on a tab or button
+class: The CSS class applied to a tiddler when rendering it - see [[Custom styles by user-class]].  Also used for [[Modals]]
 code-body: The view template will display the tiddler as code if set to ''yes''
 color: The CSS color value associated with a tiddler
 component: The name of the component responsible for an [[alert tiddler|AlertMechanism]]
@@ -29,6 +30,7 @@ name: The human readable name associated with a plugin tiddler
 parent-plugin: For a plugin, specifies which plugin of which it is a sub-plugin
 plugin-priority: A numerical value indicating the priority of a plugin tiddler
 plugin-type: The type of plugin in a plugin tiddler
+stability: The development status of a plugin: deprecated, experimental, stable, or legacy
 revision: The revision of the tiddler held at the server
 released: Date of a TiddlyWiki release
 source: The source URL associated with a tiddler

core/language/en-GB/Help/commands.tid

@@ -10,7 +10,7 @@ Sequentially run the command tokens returned from a filter
 Examples
 
 ```
---commands "[enlist{$:/build-commands-as-text}]"
+--commands "[enlist:raw{$:/build-commands-as-text}]"
 ```
 
 ```

core/language/en-GB/Help/savewikifolder.tid

@@ -19,7 +19,7 @@ The following options are supported:
 ** ''yes'' will "explode" plugins into separate tiddler files and save them to the plugin directory within the wiki folder
 ** ''no'' will suppress exploding plugins into their constituent tiddler files. It will save the plugin as a single JSON tiddler in the tiddlers folder
 
-Note that both ''explodePlugins'' options will produce wiki folders that build the same exact same original wiki. The difference lies in how plugins are represented in the wiki folder.
+Note that both ''explodePlugins'' options will produce wiki folders that build the exact same original wiki. The difference lies in how plugins are represented in the wiki folder.
 
 A common usage is to convert a TiddlyWiki HTML file into a wiki folder:
 

core/language/en-GB/Misc.multids

@@ -30,7 +30,7 @@ Error/DeserializeOperator/UnknownDeserializer: Filter Error: Unknown deserialize
 Error/Filter: Filter error
 Error/FilterSyntax: Syntax error in filter expression
 Error/FilterRunPrefix: Filter Error: Unknown prefix for filter run
-Error/IsFilterOperator: Filter Error: Unknown operand for the 'is' filter operator
+Error/IsFilterOperator: Filter Error: Unknown parameter for the 'is' filter operator
 Error/FormatFilterOperator: Filter Error: Unknown suffix for the 'format' filter operator
 Error/LoadingPluginLibrary: Error loading plugin library
 Error/NetworkErrorAlert: `<h2>''Network Error''</h2>It looks like the connection to the server has been lost. This may indicate a problem with your network connection. Please attempt to restore network connectivity before continuing.<br><br>''Any unsaved changes will be automatically synchronised when connectivity is restored''.`
@@ -70,7 +70,7 @@ No: No
 OfficialPluginLibrary: Official ~TiddlyWiki Plugin Library
 OfficialPluginLibrary/Hint: The official ~TiddlyWiki plugin library at tiddlywiki.com. Plugins, themes and language packs are maintained by the core team.
 PageTemplate/Description: the default ~TiddlyWiki layout
-PageTemplate/Name: Default ~PageTemplate
+PageTemplate/Name: Standard Layout
 PluginReloadWarning: Please save {{$:/core/ui/Buttons/save-wiki}} and reload {{$:/core/ui/Buttons/refresh}} to allow changes to ~JavaScript plugins to take effect
 RecentChanges/DateFormat: DDth MMM YYYY
 Shortcuts/Input/AdvancedSearch/Hint: Open the ~AdvancedSearch panel from within the sidebar search field

core/language/en-GB/Types/image_svg_xml.tid

@@ -1,5 +1,5 @@
 title: $:/language/Docs/Types/image/svg+xml
-description: Structured Vector Graphics image
+description: SVG image
 name: image/svg+xml
 group: Image
 group-sort: 1

core/language/en-GB/Types/image_x-icon.tid

@@ -1,5 +1,5 @@
 title: $:/language/Docs/Types/image/x-icon
-description: ICO format icon file
+description: ICO icon
 name: image/x-icon
 group: Image
 group-sort: 1

core/modules/commands/listen.js

@@ -18,7 +18,7 @@ exports.info = {
 	name: "listen",
 	synchronous: true,
 	namedParameterMode: true,
-	mandatoryParameters: [],
+	mandatoryParameters: []
 };
 
 var Command = function(params,commander,callback) {

core/modules/commands/makelibrary.js

@@ -27,33 +27,8 @@ var Command = function(params,commander,callback) {
 
 Command.prototype.execute = function() {
 	var wiki = this.commander.wiki,
-		fs = require("fs"),
-		path = require("path"),
 		upgradeLibraryTitle = this.params[0] || UPGRADE_LIBRARY_TITLE,
-		tiddlers = {};
+		tiddlers = $tw.utils.getAllPlugins();
-	// Collect up the library plugins
-	var collectPlugins = function(folder) {
-			var pluginFolders = $tw.utils.getSubdirectories(folder) || [];
-			for(var p=0; p<pluginFolders.length; p++) {
-				if(!$tw.boot.excludeRegExp.test(pluginFolders[p])) {
-					pluginFields = $tw.loadPluginFolder(path.resolve(folder,"./" + pluginFolders[p]));
-					if(pluginFields && pluginFields.title) {
-						tiddlers[pluginFields.title] = pluginFields;
-					}
-				}
-			}
-		},
-		collectPublisherPlugins = function(folder) {
-			var publisherFolders = $tw.utils.getSubdirectories(folder) || [];
-			for(var t=0; t<publisherFolders.length; t++) {
-				if(!$tw.boot.excludeRegExp.test(publisherFolders[t])) {
-					collectPlugins(path.resolve(folder,"./" + publisherFolders[t]));
-				}
-			}
-		};
-	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.pluginsPath,$tw.config.pluginsEnvVar),collectPublisherPlugins);
-	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.themesPath,$tw.config.themesEnvVar),collectPublisherPlugins);
-	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.languagesPath,$tw.config.languagesEnvVar),collectPlugins);
 	// Save the upgrade library tiddler
 	var pluginFields = {
 		title: upgradeLibraryTitle,

core/modules/commands/save.js

@@ -43,7 +43,9 @@ Saves individual tiddlers in their raw text or binary format to the specified fi
 						directory: path.resolve(self.commander.outputPath),
 						pathFilters: [filenameFilter],
 						wiki: wiki,
-						fileInfo: {}
+						fileInfo: {
+							overwrite: true
+						}
 					});
 					if(self.commander.verbose) {
 						console.log("Saving \"" + title + "\" to \"" + fileInfo.filepath + "\"");

core/modules/commands/savetiddlers.js

@@ -46,7 +46,7 @@ Command.prototype.execute = function() {
 			type = tiddler.fields.type || "text/vnd.tiddlywiki",
 			contentTypeInfo = $tw.config.contentTypeInfo[type] || {encoding: "utf8"},
 			filename = path.resolve(pathname,$tw.utils.encodeURIComponentExtended(title));
-		fs.writeFileSync(filename,tiddler.fields.text,contentTypeInfo.encoding);
+		fs.writeFileSync(filename,tiddler.fields.text || "",contentTypeInfo.encoding);
 	});
 	return null;
 };

core/modules/commands/savewikifolder.js

@@ -176,7 +176,10 @@ WikiFolderMaker.prototype.saveCustomPlugin = function(pluginTiddler) {
 	this.saveJSONFile(directory + path.sep + "plugin.info",pluginInfo);
 	self.log("Writing " + directory + path.sep + "plugin.info: " + JSON.stringify(pluginInfo,null,$tw.config.preferences.jsonSpaces));
 	var pluginTiddlers = $tw.utils.parseJSONSafe(pluginTiddler.fields.text).tiddlers; // A hashmap of tiddlers in the plugin
-	$tw.utils.each(pluginTiddlers,function(tiddler) {
+	$tw.utils.each(pluginTiddlers,function(tiddler,title) {
+		if(!tiddler.title) {
+			tiddler.title = title;
+		 }
 		self.saveTiddler(directory,new $tw.Tiddler(tiddler));
 	});
 };

core/modules/filters/all.js

@@ -28,12 +28,8 @@ function getAllFilterOperators() {
 Export our filter function
 */
 exports.all = function(source,operator,options) {
-	// Get our suboperators
-	var allFilterOperators = getAllFilterOperators();
-	// Cycle through the suboperators accumulating their results
-	var results = new $tw.utils.LinkedList(),
-		subops = operator.operand.split("+");
 	// Check for common optimisations
+	var subops = operator.operand.split("+");
 	if(subops.length === 1 && subops[0] === "") {
 		return source;
 	} else if(subops.length === 1 && subops[0] === "tiddlers") {
@@ -46,6 +42,10 @@ exports.all = function(source,operator,options) {
 		return options.wiki.eachShadowPlusTiddlers;
 	}
 	// Do it the hard way
+	// Get our suboperators
+	var allFilterOperators = getAllFilterOperators();
+	// Cycle through the suboperators accumulating their results
+	var results = new $tw.utils.LinkedList();
 	for(var t=0; t<subops.length; t++) {
 		var subop = allFilterOperators[subops[t]];
 		if(subop) {

core/modules/filters/backtranscludes.js

@@ -0,0 +1,26 @@
+/*\
+title: $:/core/modules/filters/backtranscludes.js
+type: application/javascript
+module-type: filteroperator
+
+Filter operator for returning all the backtranscludes from a tiddler
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Export our filter function
+*/
+exports.backtranscludes = function(source,operator,options) {
+	var results = [];
+	source(function(tiddler,title) {
+		$tw.utils.pushTop(results,options.wiki.getTiddlerBacktranscludes(title));
+	});
+	return results;
+};
+
+})();

core/modules/filters/crypto.js

@@ -14,12 +14,9 @@ Filter operators for cryptography, using the Stanford JavaScript library
 
 exports.sha256 = function(source,operator,options) {
 	var results = [],
-		length = parseInt(operator.operand,10) || 20,
+		length = parseInt(operator.operand,10) || 20;
-		sha256 = function(text) {
-			return sjcl.codec.hex.fromBits(sjcl.hash.sha256.hash(text)).substr(0,length);
-		};
 	source(function(tiddler,title) {
-		results.push(sha256(title));
+		results.push($tw.utils.sha256(title,{length: length}));
 	});
 	return results;
 };

core/modules/filters/encodings.js

@@ -18,16 +18,20 @@ Export our filter functions
 
 exports.decodebase64 = function(source,operator,options) {
 	var results = [];
+	var binary = operator.suffixes && operator.suffixes.indexOf("binary") !== -1;
+	var urlsafe = operator.suffixes && operator.suffixes.indexOf("urlsafe") !== -1;
 	source(function(tiddler,title) {
-		results.push($tw.utils.base64Decode(title));
+		results.push($tw.utils.base64Decode(title,binary,urlsafe));
 	});
 	return results;
 };
 
 exports.encodebase64 = function(source,operator,options) {
 	var results = [];
+	var binary = operator.suffixes && operator.suffixes.indexOf("binary") !== -1;
+	var urlsafe = operator.suffixes && operator.suffixes.indexOf("urlsafe") !== -1;
 	source(function(tiddler,title) {
-		results.push($tw.utils.base64Encode(title));
+		results.push($tw.utils.base64Encode(title,binary,urlsafe));
 	});
 	return results;
 };

core/modules/filters/json-ops.js

@@ -68,6 +68,54 @@ exports["jsontype"] = function(source,operator,options) {
 	return results;
 };
 
+exports["jsonset"] = function(source,operator,options) {
+	var suffixes = operator.suffixes || [],
+		type = suffixes[0] && suffixes[0][0],
+		indexes = operator.operands.slice(0,-1),
+		value = operator.operands[operator.operands.length - 1],
+		results = [];
+	if(operator.operands.length === 1 && operator.operands[0] === "") {
+		value = undefined; // Prevents the value from being assigned
+	}
+	switch(type) {
+		case "string":
+			// Use value unchanged
+			break;
+		case "boolean":
+			value = (value === "true" ? true : (value === "false" ? false : undefined));
+			break;
+		case "number":
+			value = $tw.utils.parseNumber(value);
+			break;
+		case "array":
+			indexes = operator.operands;
+			value = [];
+			break;
+		case "object":
+			indexes = operator.operands;
+			value = {};
+			break;
+		case "null":
+			indexes = operator.operands;
+			value = null;
+			break;
+		case "json":
+			value = $tw.utils.parseJSONSafe(value,function() {return undefined;});
+			break;
+		default:
+			// Use value unchanged
+			break;
+	}
+	source(function(tiddler,title) {
+		var data = $tw.utils.parseJSONSafe(title,title);
+		if(data) {
+			data = setDataItem(data,indexes,value);
+			results.push(JSON.stringify(data));
+		}
+	});
+	return results;
+};
+
 /*
 Given a JSON data structure and an array of index strings, return an array of the string representation of the values at the end of the index chain, or "undefined" if any of the index strings are invalid
 */
@@ -165,6 +213,18 @@ function getDataItemType(data,indexes) {
 	}
 }
 
+function getItemAtIndex(item,index) {
+	if($tw.utils.hop(item,index)) {
+		return item[index];
+	} else if($tw.utils.isArray(item)) {
+		index = $tw.utils.parseInt(index);
+		if(index < 0) { index = index + item.length };
+		return item[index]; // Will be undefined if index was out-of-bounds
+	} else {
+		return undefined;
+	}
+}
+
 /*
 Given a JSON data structure and an array of index strings, return the value at the end of the index chain, or "undefined" if any of the index strings are invalid
 */
@@ -177,7 +237,7 @@ function getDataItem(data,indexes) {
 	for(var i=0; i<indexes.length; i++) {
 		if(item !== undefined) {
 			if(item !== null && ["number","string","boolean"].indexOf(typeof item) === -1) {
-				item = item[indexes[i]];
+				item = getItemAtIndex(item,indexes[i]);
 			} else {
 				item = undefined;
 			}
@@ -186,5 +246,39 @@ function getDataItem(data,indexes) {
 	return item;
 }
 
+/*
+Given a JSON data structure, an array of index strings and a value, return the data structure with the value added at the end of the index chain. If any of the index strings are invalid then the JSON data structure is returned unmodified. If the root item is targetted then a different data object will be returned
+*/
+function setDataItem(data,indexes,value) {
+	// Ignore attempts to assign undefined
+	if(value === undefined) {
+		return data;
+	}
+	// Check for the root item
+	if(indexes.length === 0 || (indexes.length === 1 && indexes[0] === "")) {
+		return value;
+	}
+	// Traverse the JSON data structure using the index chain
+	var current = data;
+	for(var i = 0; i < indexes.length - 1; i++) {
+		current = getItemAtIndex(current,indexes[i]);
+		if(current === undefined) {
+			// Return the original JSON data structure if any of the index strings are invalid
+			return data;
+		}
+	}
+	// Add the value to the end of the index chain
+	var lastIndex = indexes[indexes.length - 1];
+	if($tw.utils.isArray(current)) {
+		lastIndex = $tw.utils.parseInt(lastIndex);
+		if(lastIndex < 0) { lastIndex = lastIndex + current.length };
+	}
+	// Only set indexes on objects and arrays
+	if(typeof current === "object") {
+		current[lastIndex] = value;
+	}
+	return data;
+}
+
 })();
 	

core/modules/filters/listops.js

@@ -58,6 +58,7 @@ Last entry/entries in list
 exports.last = function(source,operator,options) {
 	var count = $tw.utils.getInt(operator.operand,1),
 		results = [];
+	if(count === 0) return results;
 	source(function(tiddler,title) {
 		results.push(title);
 	});

core/modules/filters/strings.js

@@ -217,7 +217,10 @@ exports.splitregexp = function(source,operator,options) {
 		return ["RegExp error: " + ex];
 	}
 	source(function(tiddler,title) {
-		Array.prototype.push.apply(result,title.split(regExp));
+		var parts = title.split(regExp).map(function(part){
+			return part || "";	// make sure it's a string
+		});
+		Array.prototype.push.apply(result,parts);
 	});
 	return result;
 };

core/modules/filters/transcludes.js

@@ -0,0 +1,26 @@
+/*\
+title: $:/core/modules/filters/transcludes.js
+type: application/javascript
+module-type: filteroperator
+
+Filter operator for returning all the transcludes from a tiddler
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Export our filter function
+*/
+exports.transcludes = function(source,operator,options) {
+	var results = new $tw.utils.LinkedList();
+	source(function(tiddler,title) {
+		results.pushTop(options.wiki.getTiddlerTranscludes(title));
+	});
+	return results.toArray();
+};
+
+})();

core/modules/filters/x-listops.js

@@ -202,7 +202,7 @@ Extended filter operators to manipulate the current list.
 		}
 		if(resultsIndex !== -1) {
 			i = i + step;
-			nextOperandIndex = (i < opLength ? i : i - opLength);
+			nextOperandIndex = (i < opLength ? i : i % opLength);
 			if(operands.length > 1) {
 				results.splice(resultsIndex,1,operands[nextOperandIndex]);
 			} else {

core/modules/indexers/back-indexer.js

@@ -0,0 +1,122 @@
+/*\
+title: $:/core/modules/indexers/back-indexer.js
+type: application/javascript
+module-type: indexer
+
+By parsing the tiddler text, indexes the tiddlers' back links, back transclusions, block level back links.
+
+\*/
+function BackIndexer(wiki) {
+	this.wiki = wiki;
+}
+
+BackIndexer.prototype.init = function() {
+	this.subIndexers = {
+		link: new BackSubIndexer(this,"extractLinks"),
+		transclude: new BackSubIndexer(this,"extractTranscludes"),
+	};
+};
+
+BackIndexer.prototype.rebuild = function() {
+	$tw.utils.each(this.subIndexers,function(subIndexer) {
+		subIndexer.rebuild();
+	});
+};
+
+BackIndexer.prototype.update = function(updateDescriptor) {
+	$tw.utils.each(this.subIndexers,function(subIndexer) {
+		subIndexer.update(updateDescriptor);
+	});
+};
+function BackSubIndexer(indexer,extractor) {
+	this.wiki = indexer.wiki;
+	this.indexer = indexer;
+	this.extractor = extractor;
+	/**
+	 * {
+	 *   [target title, e.g. tiddler title being linked to]:
+	 *     {
+	 * 		   [source title, e.g. tiddler title that has link syntax in its text]: true
+	 * 	   }
+	 * }
+	 */
+	this.index = null;
+}
+
+BackSubIndexer.prototype.init = function() {
+	// lazy init until first lookup
+	this.index = null;
+}
+
+BackSubIndexer.prototype._init = function() {
+	this.index = Object.create(null);
+	var self = this;
+	this.wiki.forEachTiddler(function(sourceTitle,tiddler) {
+		var newTargets = self._getTarget(tiddler);
+		$tw.utils.each(newTargets, function(target) {
+			if(!self.index[target]) {
+				self.index[target] = Object.create(null);
+			}
+			self.index[target][sourceTitle] = true;
+		});
+	});
+}
+
+BackSubIndexer.prototype.rebuild = function() {
+	this.index = null;
+}
+
+/*
+* Get things that is being referenced in the text, e.g. tiddler names in the link syntax.
+*/
+BackSubIndexer.prototype._getTarget = function(tiddler) {
+	if(this.wiki.isBinaryTiddler(tiddler.fields.text)) {
+		return [];
+	}
+	var parser = this.wiki.parseText(tiddler.fields.type, tiddler.fields.text, {});
+	if(parser) {
+		return this.wiki[this.extractor](parser.tree);
+	}
+	return [];
+}
+
+BackSubIndexer.prototype.update = function(updateDescriptor) {
+	// lazy init/update until first lookup
+	if(!this.index) {
+		return;
+	}
+	var newTargets = [],
+	    oldTargets = [],
+	    self = this;
+	if(updateDescriptor.old.exists) {
+		oldTargets = this._getTarget(updateDescriptor.old.tiddler);
+	}
+	if(updateDescriptor.new.exists) {
+		newTargets = this._getTarget(updateDescriptor.new.tiddler);
+	}
+
+	$tw.utils.each(oldTargets,function(target) {
+		if(self.index[target]) {
+			delete self.index[target][updateDescriptor.old.tiddler.fields.title];
+		}
+	});
+	$tw.utils.each(newTargets,function(target) {
+		if(!self.index[target]) {
+			self.index[target] = Object.create(null);
+		}
+		self.index[target][updateDescriptor.new.tiddler.fields.title] = true;
+	});
+}
+
+BackSubIndexer.prototype.lookup = function(title) {
+	if(!this.index) {
+		this._init();
+	}
+	if(this.index[title]) {
+		return Object.keys(this.index[title]);
+	} else {
+		return [];
+	}
+}
+
+exports.BackIndexer = BackIndexer;

core/modules/indexers/backlinks-index.js

@@ -1,86 +0,0 @@
-/*\
-title: $:/core/modules/indexers/backlinks-indexer.js
-type: application/javascript
-module-type: indexer
-
-Indexes the tiddlers' backlinks
-
-\*/
-(function(){
-
-/*jslint node: true, browser: true */
-/*global modules: false */
-"use strict";
-
-
-function BacklinksIndexer(wiki) {
-	this.wiki = wiki;
-}
-
-BacklinksIndexer.prototype.init = function() {
-	this.index = null;
-}
-
-BacklinksIndexer.prototype.rebuild = function() {
-	this.index = null;
-}
-
-BacklinksIndexer.prototype._getLinks = function(tiddler) {
-	var parser =  this.wiki.parseText(tiddler.fields.type, tiddler.fields.text, {});
-	if(parser) {
-		return this.wiki.extractLinks(parser.tree);
-	}
-	return [];
-}
-
-BacklinksIndexer.prototype.update = function(updateDescriptor) {
-	if(!this.index) {
-		return;
-	}
-	var newLinks = [],
-	    oldLinks = [],
-	    self = this;
-	if(updateDescriptor.old.exists) {
-		oldLinks = this._getLinks(updateDescriptor.old.tiddler);
-	}
-	if(updateDescriptor.new.exists) {
-		newLinks = this._getLinks(updateDescriptor.new.tiddler);
-	}
-
-	$tw.utils.each(oldLinks,function(link) {
-		if(self.index[link]) {
-			delete self.index[link][updateDescriptor.old.tiddler.fields.title];
-		}
-	});
-	$tw.utils.each(newLinks,function(link) {
-		if(!self.index[link]) {
-			self.index[link] = Object.create(null);
-		}
-		self.index[link][updateDescriptor.new.tiddler.fields.title] = true;
-	});
-}
-
-BacklinksIndexer.prototype.lookup = function(title) {
-	if(!this.index) {
-		this.index = Object.create(null);
-		var self = this;
-		this.wiki.forEachTiddler(function(title,tiddler) {
-			var links = self._getLinks(tiddler);
-			$tw.utils.each(links, function(link) {
-				if(!self.index[link]) {
-					self.index[link] = Object.create(null);
-				}
-				self.index[link][title] = true;
-			});
-		});
-	}
-	if(this.index[title]) {
-		return Object.keys(this.index[title]);
-	} else {
-		return [];
-	}
-}
-
-exports.BacklinksIndexer = BacklinksIndexer;
-
-})();

core/modules/macros/csvtiddlers.js

@@ -35,12 +35,14 @@ exports.run = function(filter,format) {
 	// Collect all the fields
 	for(t=0;t<tiddlers.length; t++) {
 		tiddler = this.wiki.getTiddler(tiddlers[t]);
+		if(tiddler) {
 			for(f in tiddler.fields) {
 				if(fields.indexOf(f) === -1) {
 					fields.push(f);
 				}
 			}
 		}
+	}
 	// Sort the fields and bring the standard ones to the front
 	fields.sort();
 	"title text modified modifier created creator".split(" ").reverse().forEach(function(value,index) {
@@ -60,9 +62,11 @@ exports.run = function(filter,format) {
 	for(var t=0;t<tiddlers.length; t++) {
 		row = [];
 		tiddler = this.wiki.getTiddler(tiddlers[t]);
+			if(tiddler) {
 				for(f=0; f<fields.length; f++) {
 					row.push(quoteAndEscape(tiddler ? tiddler.getFieldString(fields[f]) || "" : ""));
 				}	
+			}
 		output.push(row.join(","));
 	}
 	return output.join("\n");

core/modules/parsers/textparser.js

@@ -14,10 +14,12 @@ The plain text parser processes blocks of source text into a degenerate parse tr
 
 var TextParser = function(type,text,options) {
 	this.tree = [{
-		type: "codeblock",
+		type: "genesis",
 		attributes: {
-			code: {type: "string", value: text},
+			$type: {name: "$type", type: "string", value: "$codeblock"},
-			language: {type: "string", value: type}
+			code: {name: "code", type: "string", value: text},
+			language: {name: "language", type: "string", value: type},
+			$remappable: {name: "$remappable", type:"string", value: "no"}
 		}
 	}];
 	this.source = text;
@@ -32,4 +34,3 @@ exports["text/css"] = TextParser;
 exports["application/x-tiddler-dictionary"] = TextParser;
 
 })();
-

core/modules/parsers/wikiparser/rules/codeblock.js

@@ -29,13 +29,16 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	var reEnd = /(\r?\n```$)/mg;
+	var languageStart = this.parser.pos + 3,
+		languageEnd = languageStart + this.match[1].length;
 	// Move past the match
 	this.parser.pos = this.matchRegExp.lastIndex;
 
 	// Look for the end of the block
 	reEnd.lastIndex = this.parser.pos;
 	var match = reEnd.exec(this.parser.source),
-		text;
+		text,
+		codeStart = this.parser.pos;
 	// Process the block
 	if(match) {
 		text = this.parser.source.substring(this.parser.pos,match.index);
@@ -48,8 +51,8 @@ exports.parse = function() {
 	return [{
 			type: "codeblock",
 			attributes: {
-					code: {type: "string", value: text},
+					code: {type: "string", value: text, start: codeStart, end: this.parser.pos},
-					language: {type: "string", value: this.match[1]}
+					language: {type: "string", value: this.match[1], start: languageStart, end: languageEnd}
 			}
 	}];
 };

core/modules/parsers/wikiparser/rules/codeinline.js

@@ -33,7 +33,8 @@ exports.parse = function() {
 	// Look for the end marker
 	reEnd.lastIndex = this.parser.pos;
 	var match = reEnd.exec(this.parser.source),
-		text;
+		text,
+		start = this.parser.pos;
 	// Process the text
 	if(match) {
 		text = this.parser.source.substring(this.parser.pos,match.index);
@@ -47,7 +48,9 @@ exports.parse = function() {
 		tag: "code",
 		children: [{
 			type: "text",
-			text: text
+			text: text,
+			start: start,
+			end: this.parser.pos
 		}]
 	}];
 };

core/modules/parsers/wikiparser/rules/conditional.js

@@ -0,0 +1,120 @@
+/*\
+title: $:/core/modules/parsers/wikiparser/rules/conditional.js
+type: application/javascript
+module-type: wikirule
+
+Conditional shortcut syntax
+
+```
+This is a <% if [{something}] %>Elephant<% elseif [{else}] %>Pelican<% else %>Crocodile<% endif %>
+```
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.name = "conditional";
+exports.types = {inline: true, block: true};
+
+exports.init = function(parser) {
+	this.parser = parser;
+	// Regexp to match
+	this.matchRegExp = /\<\%\s*if\s+/mg;
+	this.terminateIfRegExp = /\%\>/mg;
+};
+
+exports.findNextMatch = function(startPos) {
+	// Look for the next <% if shortcut
+	this.matchRegExp.lastIndex = startPos;
+	this.match = this.matchRegExp.exec(this.parser.source);
+	// If not found then return no match
+	if(!this.match) {
+		return undefined;
+	}
+	// Check for the next %>
+	this.terminateIfRegExp.lastIndex = this.match.index;
+	this.terminateIfMatch = this.terminateIfRegExp.exec(this.parser.source);
+	// If not found then return no match
+	if(!this.terminateIfMatch) {
+		return undefined;
+	}
+	// Return the position at which the construction was found
+	return this.match.index;
+};
+
+/*
+Parse the most recent match
+*/
+exports.parse = function() {
+	// Get the filter condition
+	var filterCondition = this.parser.source.substring(this.match.index + this.match[0].length,this.terminateIfMatch.index);
+	// Advance the parser position to past the %>
+	this.parser.pos = this.terminateIfMatch.index + this.terminateIfMatch[0].length;
+	// Parse the if clause
+	return this.parseIfClause(filterCondition);
+};
+
+exports.parseIfClause = function(filterCondition) {
+	// Create the list widget
+	var listWidget = {
+		type: "list",
+		tag: "$list",
+		isBlock: this.is.block,
+		children: [
+			{
+				type: "list-template",
+				tag: "$list-template"
+			},
+			{
+				type: "list-empty",
+				tag: "$list-empty"
+			}
+		]
+	};
+	$tw.utils.addAttributeToParseTreeNode(listWidget,"filter",filterCondition);
+	$tw.utils.addAttributeToParseTreeNode(listWidget,"variable","condition");
+	$tw.utils.addAttributeToParseTreeNode(listWidget,"limit","1");
+	// Check for an immediately following double linebreak
+	var hasLineBreak = !!$tw.utils.parseTokenRegExp(this.parser.source,this.parser.pos,/([^\S\n\r]*\r?\n(?:[^\S\n\r]*\r?\n|$))/g);
+	// Parse the body looking for else or endif
+	var reEndString = "\\<\\%\\s*(endif)\\s*\\%\\>|\\<\\%\\s*(else)\\s*\\%\\>|\\<\\%\\s*(elseif)\\s+([\\s\\S]+?)\\%\\>",
+		ex;
+	if(hasLineBreak) {
+		ex = this.parser.parseBlocksTerminatedExtended(reEndString);
+	} else {
+		var reEnd = new RegExp(reEndString,"mg");
+		ex = this.parser.parseInlineRunTerminatedExtended(reEnd,{eatTerminator: true});
+	}
+	// Put the body into the list template
+	listWidget.children[0].children = ex.tree;
+	// Check for an else or elseif
+	if(ex.match) {
+		if(ex.match[1] === "endif") {
+			// Nothing to do if we just found an endif
+		} else if(ex.match[2] === "else") {
+			// Check for an immediately following double linebreak
+			hasLineBreak = !!$tw.utils.parseTokenRegExp(this.parser.source,this.parser.pos,/([^\S\n\r]*\r?\n(?:[^\S\n\r]*\r?\n|$))/g);
+			// If we found an else then we need to parse the body looking for the endif
+			var reEndString = "\\<\\%\\s*(endif)\\s*\\%\\>",
+			ex;
+			if(hasLineBreak) {
+				ex = this.parser.parseBlocksTerminatedExtended(reEndString);
+			} else {
+				var reEnd = new RegExp(reEndString,"mg");
+				ex = this.parser.parseInlineRunTerminatedExtended(reEnd,{eatTerminator: true});
+			}
+			// Put the parsed content inside the list empty template
+			listWidget.children[1].children = ex.tree;
+		} else if(ex.match[3] === "elseif") {
+			// Parse the elseif clause by reusing this parser, passing the new filter condition
+			listWidget.children[1].children = this.parseIfClause(ex.match[4]);
+		}
+	}
+	// Return the parse tree node
+	return [listWidget];
+};
+
+})();

core/modules/parsers/wikiparser/rules/extlink.js

@@ -31,6 +31,7 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
+    var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(this.match[0].substr(0,1) === "~") {
@@ -46,7 +47,7 @@ exports.parse = function() {
 				rel: {type: "string", value: "noopener noreferrer"}
 			},
 			children: [{
-				type: "text", text: this.match[0]
+				type: "text", text: this.match[0], start: start, end: this.parser.pos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/filteredtranscludeblock.js

@@ -31,6 +31,16 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
+	var filterStart = this.parser.pos + 3;
+	var filterEnd = filterStart + this.match[1].length;
+	var toolTipStart = filterEnd + 1;
+	var toolTipEnd = toolTipStart + (this.match[2] ? this.match[2].length : 0);
+	var templateStart = toolTipEnd + 2;
+	var templateEnd = templateStart + (this.match[3] ? this.match[3].length : 0);
+	var styleStart = templateEnd + 2;
+	var styleEnd = styleStart + (this.match[4] ? this.match[4].length : 0);
+	var classesStart = styleEnd + 1;
+	var classesEnd = classesStart + (this.match[5] ? this.match[5].length : 0);
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Get the match details
 	var filter = this.match[1],
@@ -42,21 +52,21 @@ exports.parse = function() {
 	var node = {
 		type: "list",
 		attributes: {
-			filter: {type: "string", value: filter}
+			filter: {type: "string", value: filter, start: filterStart, end: filterEnd},
 		},
 		isBlock: true
 	};
 	if(tooltip) {
-		node.attributes.tooltip = {type: "string", value: tooltip};
+		node.attributes.tooltip = {type: "string", value: tooltip, start: toolTipStart, end: toolTipEnd};
 	}
 	if(template) {
-		node.attributes.template = {type: "string", value: template};
+		node.attributes.template = {type: "string", value: template, start: templateStart, end: templateEnd};
 	}
 	if(style) {
-		node.attributes.style = {type: "string", value: style};
+		node.attributes.style = {type: "string", value: style, start: styleStart, end: styleEnd};
 	}
 	if(classes) {
-		node.attributes.itemClass = {type: "string", value: classes.split(".").join(" ")};
+		node.attributes.itemClass = {type: "string", value: classes.split(".").join(" "), start: classesStart, end: classesEnd};
 	}
 	return [node];
 };

core/modules/parsers/wikiparser/rules/filteredtranscludeinline.js

@@ -30,6 +30,16 @@ exports.init = function(parser) {
 };
 
 exports.parse = function() {
+	var filterStart = this.parser.pos + 3;
+	var filterEnd = filterStart + this.match[1].length;
+	var toolTipStart = filterEnd + 1;
+	var toolTipEnd = toolTipStart + (this.match[2] ? this.match[2].length : 0);
+	var templateStart = toolTipEnd + 2;
+	var templateEnd = templateStart + (this.match[3] ? this.match[3].length : 0);
+	var styleStart = templateEnd + 2;
+	var styleEnd = styleStart + (this.match[4] ? this.match[4].length : 0);
+	var classesStart = styleEnd + 1;
+	var classesEnd = classesStart + (this.match[5] ? this.match[5].length : 0);
 	// Move past the match
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Get the match details
@@ -42,20 +52,20 @@ exports.parse = function() {
 	var node = {
 		type: "list",
 		attributes: {
-			filter: {type: "string", value: filter}
+			filter: {type: "string", value: filter, start: filterStart, end: filterEnd},
 		}
 	};
 	if(tooltip) {
-		node.attributes.tooltip = {type: "string", value: tooltip};
+		node.attributes.tooltip = {type: "string", value: tooltip, start: toolTipStart, end: toolTipEnd};
 	}
 	if(template) {
-		node.attributes.template = {type: "string", value: template};
+		node.attributes.template = {type: "string", value: template, start: templateStart, end: templateEnd};
 	}
 	if(style) {
-		node.attributes.style = {type: "string", value: style};
+		node.attributes.style = {type: "string", value: style, start: styleStart, end: styleEnd};
 	}
 	if(classes) {
-		node.attributes.itemClass = {type: "string", value: classes.split(".").join(" ")};
+		node.attributes.itemClass = {type: "string", value: classes.split(".").join(" "), start: classesStart, end: classesEnd};
 	}
 	return [node];
 };

core/modules/parsers/wikiparser/rules/fnprocdef.js

@@ -49,11 +49,11 @@ exports.parse = function() {
 	if(this.match[3]) {
 		params = $tw.utils.parseParameterDefinition(this.match[4]);
 	}
-	// Is this a multiline definition?
+	// Is the remainder of the line blank after the parameter close paren?
 	var reEnd;
 	if(this.match[5]) {
-		// If so, the end of the body is marked with \end
+		// If so, it is a multiline definition and the end of the body is marked with \end
-		reEnd = new RegExp("(\\r?\\n[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[2]) + ")?(?:$|\\r?\\n))","mg");
+		reEnd = new RegExp("((:?^|\\r?\\n)[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[2]) + ")?(?:$|\\r?\\n))","mg");
 	} else {
 		// Otherwise, the end of the definition is marked by the end of the line
 		reEnd = /($|\r?\n)/mg;

core/modules/parsers/wikiparser/rules/hardlinebreaks.js

@@ -45,10 +45,11 @@ exports.parse = function() {
 		reEnd.lastIndex = this.parser.pos;
 		match = reEnd.exec(this.parser.source);
 		if(match) {
+			var start = this.parser.pos;
 			this.parser.pos = reEnd.lastIndex;
 			// Add a line break if the terminator was a line break
 			if(match[2]) {
-				tree.push({type: "element", tag: "br"});
+				tree.push({type: "element", tag: "br", start: start, end: this.parser.pos});
 			}
 		}
 	} while(match && !match[1]);

core/modules/parsers/wikiparser/rules/heading.js

@@ -30,7 +30,9 @@ exports.parse = function() {
 	// Move past the !s
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Parse any classes, whitespace and then the heading itself
+	var classStart = this.parser.pos;
 	var classes = this.parser.parseClasses();
+	var classEnd = this.parser.pos;
 	this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
 	var tree = this.parser.parseInlineRun(/(\r?\n)/mg);
 	// Return the heading
@@ -38,7 +40,7 @@ exports.parse = function() {
 		type: "element",
 		tag: "h" + headingLevel,
 		attributes: {
-			"class": {type: "string", value: classes.join(" ")}
+			"class": {type: "string", value: classes.join(" "), start: classStart, end: classEnd}
 		},
 		children: tree
 	}];

core/modules/parsers/wikiparser/rules/html.js

@@ -44,6 +44,10 @@ Parse the most recent match
 exports.parse = function() {
 	// Retrieve the most recent match so that recursive calls don't overwrite it
 	var tag = this.nextTag;
+	if (!tag.isSelfClosing) {
+		tag.openTagStart = tag.start;
+		tag.openTagEnd = tag.end;
+	}
 	this.nextTag = null;
 	// Advance the parser position to past the tag
 	this.parser.pos = tag.end;
@@ -60,6 +64,27 @@ exports.parse = function() {
 			var reEnd = new RegExp("(" + reEndString + ")","mg");
 			tag.children = this.parser.parseInlineRun(reEnd,{eatTerminator: true});
 		}
+		tag.end = this.parser.pos;
+		tag.closeTagEnd = tag.end;
+		if (tag.closeTagEnd === tag.openTagEnd || this.parser.source[tag.closeTagEnd - 1] !== '>') {
+			tag.closeTagStart = tag.end;
+		} else {
+			tag.closeTagStart = tag.closeTagEnd - 2;
+			var closeTagMinPos = tag.children.length > 0 ? tag.children[tag.children.length-1].end : tag.openTagEnd;
+			if (!Number.isSafeInteger(closeTagMinPos)) closeTagMinPos = tag.openTagEnd;
+			while (tag.closeTagStart >= closeTagMinPos) {
+				var char = this.parser.source[tag.closeTagStart];
+				if (char === '>') {
+					tag.closeTagStart = -1;
+					break;
+				}
+				if (char === '<') break;
+				tag.closeTagStart -= 1;
+			}
+			if (tag.closeTagStart < closeTagMinPos) {
+				tag.closeTagStart = tag.end;
+			}
+		}
 	}
 	// Return the tag
 	return [tag];

core/modules/parsers/wikiparser/rules/image.js

@@ -122,9 +122,9 @@ exports.parseImage = function(source,pos) {
 	}
 	pos = token.end;
 	if(token.match[1]) {
-		node.attributes.tooltip = {type: "string", value: token.match[1].trim()};
+		node.attributes.tooltip = {type: "string", value: token.match[1].trim(),start: token.start,end:token.start + token.match[1].length - 1};
 	}
-	node.attributes.source = {type: "string", value: (token.match[2] || "").trim()};
+	node.attributes.source = {type: "string", value: (token.match[2] || "").trim(), start: token.start + (token.match[1] ? token.match[1].length : 0), end: token.end - 2};
 	// Update the end position
 	node.end = pos;
 	return node;

core/modules/parsers/wikiparser/rules/import.js

@@ -38,13 +38,14 @@ exports.parse = function() {
 	// Parse the filter terminated by a line break
 	var reMatch = /(.*)(?:$|\r?\n)/mg;
 	reMatch.lastIndex = this.parser.pos;
+	var filterStart = this.parser.source;
 	var match = reMatch.exec(this.parser.source);
 	this.parser.pos = reMatch.lastIndex;
 	// Parse tree nodes to return
 	return [{
 		type: "importvariables",
 		attributes: {
-			filter: {type: "string", value: match[1]}
+			filter: {type: "string", value: match[1], start: filterStart, end: this.parser.pos}
 		},
 		children: []
 	}];

core/modules/parsers/wikiparser/rules/list.js

@@ -74,6 +74,7 @@ exports.parse = function() {
 		// Match the list marker
 		var reMatch = /([\*#;:>]+)/mg;
 		reMatch.lastIndex = this.parser.pos;
+		var start = this.parser.pos;
 		var match = reMatch.exec(this.parser.source);
 		if(!match || match.index !== this.parser.pos) {
 			break;
@@ -94,9 +95,21 @@ exports.parse = function() {
 			}
 			// Construct the list element or reuse the previous one at this level
 			if(listStack.length <= t) {
-				var listElement = {type: "element", tag: listInfo.listTag, children: [
+				var listElement = {
-					{type: "element", tag: listInfo.itemTag, children: []}
+					type: "element",
-				]};
+					tag: listInfo.listTag,
+					children: [
+						{
+							type: "element",
+							tag: listInfo.itemTag,
+							children: [],
+							start: start,
+							end: this.parser.pos,
+						}
+					],
+					start: start,
+					end: this.parser.pos,
+				};
 				// Link this list element into the last child item of the parent list item
 				if(t) {
 					var prevListItem = listStack[t-1].children[listStack[t-1].children.length-1];
@@ -105,21 +118,33 @@ exports.parse = function() {
 				// Save this element in the stack
 				listStack[t] = listElement;
 			} else if(t === (match[0].length - 1)) {
-				listStack[t].children.push({type: "element", tag: listInfo.itemTag, children: []});
+				listStack[t].children.push({
+					type: "element",
+					tag: listInfo.itemTag,
+					children: [],
+					start: start,
+					end: this.parser.pos,
+				});
 			}
 		}
 		if(listStack.length > match[0].length) {
 			listStack.splice(match[0].length,listStack.length - match[0].length);
 		}
 		// Process the body of the list item into the last list item
+		var classStart = this.parser.pos;
 		var lastListChildren = listStack[listStack.length-1].children,
 			lastListItem = lastListChildren[lastListChildren.length-1],
 			classes = this.parser.parseClasses();
+		var classEnd = this.parser.pos;
 		this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
 		var tree = this.parser.parseInlineRun(/(\r?\n)/mg);
 		lastListItem.children.push.apply(lastListItem.children,tree);
+		lastListItem.end = this.parser.pos;
+		listStack[listStack.length-1].end = this.parser.pos;
 		if(classes.length > 0) {
 			$tw.utils.addClassToParseTreeNode(lastListItem,classes.join(" "));
+			lastListItem.attributes.class.start = classStart;
+			lastListItem.attributes.class.end = classEnd;
 		}
 		// Consume any whitespace following the list item
 		this.parser.skipWhitespace();

core/modules/parsers/wikiparser/rules/macrodef.js

@@ -54,11 +54,11 @@ exports.parse = function() {
 			paramMatch = reParam.exec(paramString);
 		}
 	}
-	// Is this a multiline definition?
+	// Is the remainder of the \define line blank after the parameter close paren?
 	var reEnd;
 	if(this.match[3]) {
-		// If so, the end of the body is marked with \end
+		// If so, it is a multiline definition and the end of the body is marked with \end
-		reEnd = new RegExp("(\\r?\\n[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[1]) + ")?(?:$|\\r?\\n))","mg");
+		reEnd = new RegExp("((?:^|\\r?\\n)[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[1]) + ")?(?:$|\\r?\\n))","mg");
 	} else {
 		// Otherwise, the end of the definition is marked by the end of the line
 		reEnd = /($|\r?\n)/mg;

core/modules/parsers/wikiparser/rules/prettyextlink.js

@@ -96,15 +96,20 @@ exports.parseLink = function(source,pos) {
 		splitPos = null;
 	}
 	// Pull out the tooltip and URL
-	var tooltip, URL;
+	var tooltip, URL, urlStart;
+	textNode.start = pos;
 	if(splitPos) {
+		urlStart = splitPos + 1;
 		URL = source.substring(splitPos + 1,closePos).trim();
 		textNode.text = source.substring(pos,splitPos).trim();
+		textNode.end = splitPos;
 	} else {
+		urlStart = pos;
 		URL = source.substring(pos,closePos).trim();
 		textNode.text = URL;
+		textNode.end = closePos;
 	}
-	node.attributes.href = {type: "string", value: URL};
+	node.attributes.href = {type: "string", value: URL, start: urlStart, end: closePos};
 	node.attributes.target = {type: "string", value: "_blank"};
 	node.attributes.rel = {type: "string", value: "noopener noreferrer"};
 	// Update the end position

core/modules/parsers/wikiparser/rules/prettylink.js

@@ -29,11 +29,18 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
+	var start = this.parser.pos + 2;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Process the link
 	var text = this.match[1],
 		link = this.match[2] || text,
-		anchor = this.match[3] || "";
+		anchor = this.match[3] || "",
+		textEndPos = this.parser.source.indexOf("|", start);
+	if (textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
+		textEndPos = this.matchRegExp.lastIndex - 2;
+	}
+	var linkStart = this.match[2] ? (start + this.match[1].length + 1) : start;
+	var linkEnd = linkStart + link.length;
 	if($tw.utils.isLinkExternal(link)) {
 		// add back the part after `^` to the ext link, if it happen to has one.
 		if(anchor) {
@@ -43,24 +50,24 @@ exports.parse = function() {
 			type: "element",
 			tag: "a",
 			attributes: {
-				href: {type: "string", value: link},
+				href: {type: "string", value: link, start: linkStart, end: linkEnd},
 				"class": {type: "string", value: "tc-tiddlylink-external"},
 				target: {type: "string", value: "_blank"},
 				rel: {type: "string", value: "noopener noreferrer"}
 			},
 			children: [{
-				type: "text", text: text
+				type: "text", text: text, start: start, end: textEndPos
 			}]
 		}];
 	} else {
 		return [{
 			type: "link",
 			attributes: {
-				to: {type: "string", value: link},
+				to: {type: "string", value: link, start: linkStart, end: linkEnd},
 				toAnchor: {type: "string", value: anchor},
 			},
 			children: [{
-				type: "text", text: text
+				type: "text", text: text, start: start, end: textEndPos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/quoteblock.js

@@ -3,30 +3,7 @@ title: $:/core/modules/parsers/wikiparser/rules/quoteblock.js
 type: application/javascript
 module-type: wikirule
 
-Wiki text rule for quote blocks. For example:
+Wiki text rule for quote blocks.
-
-```
-	<<<.optionalClass(es) optional cited from
-	a quote
-	<<<
-	
-	<<<.optionalClass(es)
-	a quote
-	<<< optional cited from
-```
-
-Quotes can be quoted by putting more <s
-
-```
-	<<<
-	Quote Level 1
-	
-	<<<<
-	QuoteLevel 2
-	<<<<
-	
-	<<<
-```
 
 \*/
 (function(){
@@ -47,14 +24,17 @@ exports.init = function(parser) {
 exports.parse = function() {
 	var classes = ["tc-quote"];
 	// Get all the details of the match
-	var reEndString = "^" + this.match[1] + "(?!<)";
+	var reEndString = "^\\s*" + this.match[1] + "(?!<)";
 	// Move past the <s
 	this.parser.pos = this.matchRegExp.lastIndex;
-	
 	// Parse any classes, whitespace and then the optional cite itself
+	var classStart = this.parser.pos;
 	classes.push.apply(classes, this.parser.parseClasses());
+	var classEnd = this.parser.pos;
 	this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
+	var citeStart = this.parser.pos;
 	var cite = this.parser.parseInlineRun(/(\r?\n)/mg);
+	var citeEnd = this.parser.pos;
 	// before handling the cite, parse the body of the quote
 	var tree = this.parser.parseBlocks(reEndString);
 	// If we got a cite, put it before the text
@@ -62,18 +42,24 @@ exports.parse = function() {
 		tree.unshift({
 			type: "element",
 			tag: "cite",
-			children: cite
+			children: cite,
+			start: citeStart,
+			end: citeEnd
 		});
 	}
 	// Parse any optional cite
 	this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
+	citeStart = this.parser.pos;
 	cite = this.parser.parseInlineRun(/(\r?\n)/mg);
+	citeEnd = this.parser.pos;
 	// If we got a cite, push it
 	if(cite.length > 0) {
 		tree.push({
 			type: "element",
 			tag: "cite",
-			children: cite
+			children: cite,
+			start: citeStart,
+			end: citeEnd
 		});
 	}
 	// Return the blockquote element
@@ -81,7 +67,7 @@ exports.parse = function() {
 		type: "element",
 		tag: "blockquote",
 		attributes: {
-			class: { type: "string", value: classes.join(" ") },
+			class: { type: "string", value: classes.join(" "), start: classStart, end: classEnd },
 		},
 		children: tree
 	}];

core/modules/parsers/wikiparser/rules/syslink.js

@@ -29,10 +29,11 @@ exports.init = function(parser) {
 exports.parse = function() {
 	var match = this.match[0];
 	// Move past the match
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(match.substr(0,1) === "~") {
-		return [{type: "text", text: match.substr(1)}];
+		return [{type: "text", text: match.substr(1), start: start+1, end: this.parser.pos}];
 	} else {
 		return [{
 			type: "link",
@@ -41,7 +42,9 @@ exports.parse = function() {
 			},
 			children: [{
 				type: "text",
-				text: match
+				text: match,
+				start: start,
+				end: this.parser.pos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/table.js

@@ -150,7 +150,7 @@ exports.parse = function() {
 		} else {
 			// Otherwise, create a new row if this one is of a different type
 			if(rowType !== currRowType) {
-				rowContainer = {type: "element", tag: rowContainerTypes[rowType], children: []};
+				rowContainer = {type: "element", tag: rowContainerTypes[rowType], children: [], start: this.parser.pos, end: this.parser.pos};
 				table.children.push(rowContainer);
 				currRowType = rowType;
 			}
@@ -178,6 +178,7 @@ exports.parse = function() {
 				// Increment the row count
 				rowCount++;
 			}
+			rowContainer.end = this.parser.pos;
 		}
 		rowMatch = rowRegExp.exec(this.parser.source);
 	}

core/modules/parsers/wikiparser/rules/typedblock.js

@@ -46,6 +46,7 @@ exports.parse = function() {
 		renderType = this.match[2];
 	// Move past the match
 	this.parser.pos = this.matchRegExp.lastIndex;
+	var start = this.parser.pos;
 	// Look for the end of the block
 	reEnd.lastIndex = this.parser.pos;
 	var match = reEnd.exec(this.parser.source),
@@ -74,7 +75,9 @@ exports.parse = function() {
 			tag: "pre",
 			children: [{
 				type: "text",
-				text: text
+				text: text,
+				start: start,
+				end: this.parser.pos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/wikilink.js

@@ -36,6 +36,7 @@ exports.parse = function() {
 	// Get the details of the match
 	var linkText = this.match[0];
 	// Move past the macro call
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// If the link starts with the unwikilink character then just output it as plain text
 	if(linkText.substr(0,1) === $tw.config.textPrimitives.unWikiLink) {
@@ -57,7 +58,9 @@ exports.parse = function() {
 		},
 		children: [{
 			type: "text",
-			text: linkText
+			text: linkText,
+			start: start,
+			end: this.parser.pos
 		}]
 	}];
 };

core/modules/parsers/wikiparser/wikiparser.js

@@ -91,6 +91,11 @@ var WikiParser = function(type,text,options) {
 	} else {
 		topBranch.push.apply(topBranch,this.parseBlocks());
 	}
+	// Build rules' name map
+	this.usingRuleMap = {};
+	$tw.utils.each(this.pragmaRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
+	$tw.utils.each(this.blockRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
+	$tw.utils.each(this.inlineRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
 	// Return the parse tree
 };
 
@@ -194,6 +199,7 @@ Parse any pragmas at the beginning of a block of parse text
 WikiParser.prototype.parsePragmas = function() {
 	var currentTreeBranch = this.tree;
 	while(true) {
+		var savedPos = this.pos;
 		// Skip whitespace
 		this.skipWhitespace();
 		// Check for the end of the text
@@ -204,16 +210,24 @@ WikiParser.prototype.parsePragmas = function() {
 		var nextMatch = this.findNextMatch(this.pragmaRules,this.pos);
 		// If not, just exit
 		if(!nextMatch || nextMatch.matchIndex !== this.pos) {
+			this.pos = savedPos;
 			break;
 		}
 		// Process the pragma rule
+		var start = this.pos;
 		var subTree = nextMatch.rule.parse();
 		if(subTree.length > 0) {
+			// Set the start and end positions of the pragma rule if
+			if (subTree[0].start === undefined) subTree[0].start = start;
+			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 			// Quick hack; we only cope with a single parse tree node being returned, which is true at the moment
 			currentTreeBranch.push.apply(currentTreeBranch,subTree);
 			subTree[0].children = [];
 			currentTreeBranch = subTree[0].children;
 		}
+		// Skip whitespace after the pragma
+		this.skipWhitespace();
 	}
 	return currentTreeBranch;
 };
@@ -223,7 +237,7 @@ Parse a block from the current position
 	terminatorRegExpString: optional regular expression string that identifies the end of plain paragraphs. Must not include capturing parenthesis
 */
 WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
-	var terminatorRegExp = terminatorRegExpString ? new RegExp("(" + terminatorRegExpString + "|\\r?\\n\\r?\\n)","mg") : /(\r?\n\r?\n)/mg;
+	var terminatorRegExp = terminatorRegExpString ? new RegExp(terminatorRegExpString + "|\\r?\\n\\r?\\n","mg") : /(\r?\n\r?\n)/mg;
 	this.skipWhitespace();
 	if(this.pos >= this.sourceLength) {
 		return [];
@@ -231,7 +245,15 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 	// Look for a block rule that applies at the current position
 	var nextMatch = this.findNextMatch(this.blockRules,this.pos);
 	if(nextMatch && nextMatch.matchIndex === this.pos) {
-		return nextMatch.rule.parse();
+		var start = this.pos;
+		var subTree = nextMatch.rule.parse();
+		// Set the start and end positions of the first and last blocks if they're not already set
+		if (subTree.length > 0) {
+			if (subTree[0].start === undefined) subTree[0].start = start;
+			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+		}
+		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
+		return subTree;
 	}
 	// Treat it as a paragraph if we didn't find a block rule
 	var start = this.pos;
@@ -264,11 +286,21 @@ WikiParser.prototype.parseBlocksUnterminated = function() {
 };
 
 /*
-Parse blocks of text until a terminating regexp is encountered
+Parse blocks of text until a terminating regexp is encountered. Wrapper for parseBlocksTerminatedExtended that just returns the parse tree
 */
 WikiParser.prototype.parseBlocksTerminated = function(terminatorRegExpString) {
-	var terminatorRegExp = new RegExp("(" + terminatorRegExpString + ")","mg"),
+	var ex = this.parseBlocksTerminatedExtended(terminatorRegExpString);
-		tree = [];
+	return ex.tree;
+};
+
+/*
+Parse blocks of text until a terminating regexp is encountered
+*/
+WikiParser.prototype.parseBlocksTerminatedExtended = function(terminatorRegExpString) {
+	var terminatorRegExp = new RegExp(terminatorRegExpString,"mg"),
+		result = {
+			tree: []
+		};
 	// Skip any whitespace
 	this.skipWhitespace();
 	//  Check if we've got the end marker
@@ -277,7 +309,7 @@ WikiParser.prototype.parseBlocksTerminated = function(terminatorRegExpString) {
 	// Parse the text into blocks
 	while(this.pos < this.sourceLength && !(match && match.index === this.pos)) {
 		var blocks = this.parseBlock(terminatorRegExpString);
-		tree.push.apply(tree,blocks);
+		result.tree.push.apply(result.tree,blocks);
 		// Skip any whitespace
 		this.skipWhitespace();
 		//  Check if we've got the end marker
@@ -286,8 +318,9 @@ WikiParser.prototype.parseBlocksTerminated = function(terminatorRegExpString) {
 	}
 	if(match && match.index === this.pos) {
 		this.pos = match.index + match[0].length;
+		result.match = match;
 	}
-	return tree;
+	return result;
 };
 
 /*
@@ -317,7 +350,16 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
 			this.pos = nextMatch.matchIndex;
 		}
 		// Process the run rule
-		tree.push.apply(tree,nextMatch.rule.parse());
+		var start = this.pos;
+		var subTree = nextMatch.rule.parse();
+		// Set the start and end positions of the first and last child if they're not already set
+		if (subTree.length > 0) {
+			// Set the start and end positions of the first and last child if they're not already set
+			if (subTree[0].start === undefined) subTree[0].start = start;
+			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+		}
+		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
+		tree.push.apply(tree,subTree);
 		// Look for the next run rule
 		nextMatch = this.findNextMatch(this.inlineRules,this.pos);
 	}
@@ -330,6 +372,11 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
 };
 
 WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,options) {
+	var ex = this.parseInlineRunTerminatedExtended(terminatorRegExp,options);
+	return ex.tree;
+};
+
+WikiParser.prototype.parseInlineRunTerminatedExtended = function(terminatorRegExp,options) {
 	options = options || {};
 	var tree = [];
 	// Find the next occurrence of the terminator
@@ -349,7 +396,10 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option
 				if(options.eatTerminator) {
 					this.pos += terminatorMatch[0].length;
 				}
-				return tree;
+				return {
+					match: terminatorMatch,
+					tree: tree
+				};
 			}
 		}
 		// Process any inline rule, along with the text preceding it
@@ -360,7 +410,15 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option
 				this.pos = inlineRuleMatch.matchIndex;
 			}
 			// Process the inline rule
-			tree.push.apply(tree,inlineRuleMatch.rule.parse());
+			var start = this.pos;
+			var subTree = inlineRuleMatch.rule.parse();
+			// Set the start and end positions of the first and last child if they're not already set
+			if (subTree.length > 0) {
+				if (subTree[0].start === undefined) subTree[0].start = start;
+				if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			}
+			$tw.utils.each(subTree, function (node) { node.rule = inlineRuleMatch.rule.name; });
+			tree.push.apply(tree,subTree);
 			// Look for the next inline rule
 			inlineRuleMatch = this.findNextMatch(this.inlineRules,this.pos);
 			// Look for the next terminator match
@@ -373,7 +431,9 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option
 		this.pushTextWidget(tree,this.source.substr(this.pos),this.pos,this.sourceLength);
 	}
 	this.pos = this.sourceLength;
-	return tree;
+	return {
+		tree: tree
+	};
 };
 
 /*
@@ -437,4 +497,3 @@ WikiParser.prototype.amendRules = function(type,names) {
 exports["text/vnd.tiddlywiki"] = WikiParser;
 
 })();
-

core/modules/savers/github.js

@@ -31,7 +31,7 @@ GitHubSaver.prototype.save = function(text,method,callback) {
 		headers = {
 			"Accept": "application/vnd.github.v3+json",
 			"Content-Type": "application/json;charset=UTF-8",
-			"Authorization": "Basic " + window.btoa(username + ":" + password),
+			"Authorization": "Basic " + $tw.utils.base64Encode(username + ":" + password),
 			"If-None-Match": ""
 		};
 	// Bail if we don't have everything we need

core/modules/server/server.js

@@ -140,6 +140,11 @@ function sendResponse(request,response,statusCode,headers,data,encoding) {
 				return;
 			}
 		}
+	} else {
+		// RFC 7231, 6.1. Overview of Status Codes:
+		// Browser clients may cache 200, 203, 204, 206, 300, 301, 
+		// 404, 405, 410, 414, and 501 unless given explicit cache controls
+		headers["Cache-Control"] = headers["Cache-Control"] || "no-store";
 	}
 	/*
 	If the gzip=yes is set, check if the user agent permits compression. If so,

core/modules/startup/render.js

@@ -29,7 +29,11 @@ var THROTTLE_REFRESH_TIMEOUT = 400;
 
 exports.startup = function() {
 	// Set up the title
-	$tw.titleWidgetNode = $tw.wiki.makeTranscludeWidget(PAGE_TITLE_TITLE,{document: $tw.fakeDocument, parseAsInline: true});
+	$tw.titleWidgetNode = $tw.wiki.makeTranscludeWidget(PAGE_TITLE_TITLE, {
+		document: $tw.fakeDocument,
+		parseAsInline: true,
+		importPageMacros: true,
+	});
 	$tw.titleContainer = $tw.fakeDocument.createElement("div");
 	$tw.titleWidgetNode.render($tw.titleContainer,null);
 	document.title = $tw.titleContainer.textContent;
@@ -81,6 +85,8 @@ exports.startup = function() {
 		deferredChanges = Object.create(null);
 		$tw.hooks.invokeHook("th-page-refreshed");
 	}
+	var throttledRefresh = $tw.perf.report("throttledRefresh",refresh);
+
 	// Add the change event handler
 	$tw.wiki.addEventListener("change",$tw.perf.report("mainRefresh",function(changes) {
 		// Check if only tiddlers that are throttled have changed
@@ -101,7 +107,7 @@ exports.startup = function() {
 			if(isNaN(timeout)) {
 				timeout = THROTTLE_REFRESH_TIMEOUT;
 			}
-			timerId = setTimeout(refresh,timeout);
+			timerId = setTimeout(throttledRefresh,timeout);
 			$tw.utils.extend(deferredChanges,changes);
 		} else {
 			$tw.utils.extend(deferredChanges,changes);

core/modules/startup/rootwidget.js

@@ -39,6 +39,7 @@ exports.startup = function() {
 			method: params.method,
 			body: params.body,
 			binary: params.binary,
+			useDefaultHeaders: params.useDefaultHeaders,
 			oncompletion: params.oncompletion,
 			onprogress: params.onprogress,
 			bindStatus: params["bind-status"],
@@ -47,7 +48,11 @@ exports.startup = function() {
 			headers: getPropertiesWithPrefix(params,"header-"),
 			passwordHeaders: getPropertiesWithPrefix(params,"password-header-"),
 			queryStrings: getPropertiesWithPrefix(params,"query-"),
-			passwordQueryStrings: getPropertiesWithPrefix(params,"password-query-")
+			passwordQueryStrings: getPropertiesWithPrefix(params,"password-query-"),
+			basicAuthUsername: params["basic-auth-username"],
+			basicAuthUsernameFromStore: params["basic-auth-username-from-store"],
+			basicAuthPassword: params["basic-auth-password"],
+			basicAuthPasswordFromStore: params["basic-auth-password-from-store"]
 		});
 	});
 	$tw.rootWidget.addEventListener("tm-http-cancel-all-requests",function(event) {
@@ -68,7 +73,10 @@ exports.startup = function() {
 	});
 	// Install the copy-to-clipboard  mechanism
 	$tw.rootWidget.addEventListener("tm-copy-to-clipboard",function(event) {
-		$tw.utils.copyToClipboard(event.param);
+		$tw.utils.copyToClipboard(event.param,{
+			successNotification: event.paramObject && event.paramObject.successNotification,
+			failureNotification: event.paramObject && event.paramObject.failureNotification
+		});
 	});
 	// Install the tm-focus-selector message
 	$tw.rootWidget.addEventListener("tm-focus-selector",function(event) {

core/modules/startup/story.js

@@ -93,7 +93,9 @@ exports.startup = function() {
 				updateAddressBar: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_UPDATE_ADDRESS_BAR,"yes").trim() === "yes" ? "permalink" : "none",
 				updateHistory: $tw.wiki.getTiddlerText(CONFIG_UPDATE_HISTORY,"no").trim(),
 				targetTiddler: event.param || event.tiddlerTitle,
-				copyToClipboard: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_COPY_TO_CLIPBOARD,"yes").trim() === "yes" ? "permalink" : "none"
+				copyToClipboard: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_COPY_TO_CLIPBOARD,"yes").trim() === "yes" ? "permalink" : "none",
+				successNotification: event.paramObject && event.paramObject.successNotification,
+				failureNotification: event.paramObject && event.paramObject.failureNotification
 			});
 		});
 		// Listen for the tm-permaview message
@@ -102,7 +104,9 @@ exports.startup = function() {
 				updateAddressBar: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_UPDATE_ADDRESS_BAR,"yes").trim() === "yes" ? "permaview" : "none",
 				updateHistory: $tw.wiki.getTiddlerText(CONFIG_UPDATE_HISTORY,"no").trim(),
 				targetTiddler: event.param || event.tiddlerTitle,
-				copyToClipboard: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_COPY_TO_CLIPBOARD,"yes").trim() === "yes" ? "permaview" : "none"
+				copyToClipboard: $tw.wiki.getTiddlerText(CONFIG_PERMALINKVIEW_COPY_TO_CLIPBOARD,"yes").trim() === "yes" ? "permaview" : "none",
+				successNotification: event.paramObject && event.paramObject.successNotification,
+				failureNotification: event.paramObject && event.paramObject.failureNotification
 			});
 		});
 	}
@@ -177,6 +181,8 @@ options.updateAddressBar: "permalink", "permaview" or "no" (defaults to "permavi
 options.updateHistory: "yes" or "no" (defaults to "no")
 options.copyToClipboard: "permalink", "permaview" or "no" (defaults to "no")
 options.targetTiddler: optional title of target tiddler for permalink
+options.successNotification: optional title of tiddler to use as the notification in case of success
+options.failureNotification: optional title of tiddler to use as the notification in case of failure
 */
 function updateLocationHash(options) {
 	// Get the story and the history stack
@@ -205,14 +211,18 @@ function updateLocationHash(options) {
 			break;
 	}
 	// Copy URL to the clipboard
+	var url = "";
 	switch(options.copyToClipboard) {
 		case "permalink":
-			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler));
+			url = $tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler);
 			break;
 		case "permaview":
-			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler) + ":" + encodeURIComponent($tw.utils.stringifyList(storyList)));
+			url = $tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler) + ":" + encodeURIComponent($tw.utils.stringifyList(storyList));
 			break;
 	}
+	if(url) {
+		$tw.utils.copyToClipboard(url,{successNotification: options.successNotification, failureNotification: options.failureNotification});
+	}
 	// Only change the location hash if we must, thus avoiding unnecessary onhashchange events
 	if($tw.utils.getLocationHash() !== $tw.locationHash) {
 		if(options.updateHistory === "yes") {

core/modules/startup/windows.js

@@ -52,10 +52,11 @@ exports.startup = function() {
 		$tw.windows[windowID] = srcWindow;
 		// Check for reopening the same window
 		if(srcWindow.haveInitialisedWindow) {
+			srcWindow.focus();
 			return;
 		}
 		// Initialise the document
-		srcDocument.write("<html><head></head><body class='tc-body tc-single-tiddler-window'></body></html>");
+		srcDocument.write("<!DOCTYPE html><head></head><body class='tc-body tc-single-tiddler-window'></body></html>");
 		srcDocument.close();
 		srcDocument.title = windowTitle;
 		srcWindow.addEventListener("beforeunload",function(event) {

core/modules/syncer.js

@@ -24,7 +24,7 @@ Syncer.prototype.titleSyncPollingInterval = "$:/config/SyncPollingInterval";
 Syncer.prototype.titleSyncDisableLazyLoading = "$:/config/SyncDisableLazyLoading";
 Syncer.prototype.titleSavedNotification = "$:/language/Notifications/Save/Done";
 Syncer.prototype.titleSyncThrottleInterval = "$:/config/SyncThrottleInterval";
-Syncer.prototype.taskTimerInterval = 1 * 1000; // Interval for sync timer
+Syncer.prototype.taskTimerInterval = 0.25 * 1000; // Interval for sync timer
 Syncer.prototype.throttleInterval = 1 * 1000; // Defer saving tiddlers if they've changed in the last 1s...
 Syncer.prototype.errorRetryInterval = 5 * 1000; // Interval to retry after an error
 Syncer.prototype.fallbackInterval = 10 * 1000; // Unless the task is older than 10s
@@ -74,9 +74,11 @@ function Syncer(options) {
 	this.titlesHaveBeenLazyLoaded = {}; // Hashmap of titles of tiddlers that have already been lazily loaded from the server
 	// Timers
 	this.taskTimerId = null; // Timer for task dispatch
-	this.pollTimerId = null; // Timer for polling server
 	// Number of outstanding requests
 	this.numTasksInProgress = 0;
+	// True when we want to force an immediate sync from the server
+	this.forceSyncFromServer = false;
+	this.timestampLastSyncFromServer = new Date();
 	// Listen out for changes to tiddlers
 	this.wiki.addEventListener("change",function(changes) {
 		// Filter the changes to just include ones that are being synced
@@ -203,16 +205,17 @@ Syncer.prototype.readTiddlerInfo = function() {
 Checks whether the wiki is dirty (ie the window shouldn't be closed)
 */
 Syncer.prototype.isDirty = function() {
-	this.logger.log("Checking dirty status");
+	var self = this;
+	function checkIsDirty() {
 		// Check tiddlers that are in the store and included in the filter function
-	var titles = this.getSyncedTiddlers();
+		var titles = self.getSyncedTiddlers();
 		for(var index=0; index<titles.length; index++) {
 			var title = titles[index],
-			tiddlerInfo = this.tiddlerInfo[title];
+				tiddlerInfo = self.tiddlerInfo[title];
-		if(this.wiki.tiddlerExists(title)) {
+			if(self.wiki.tiddlerExists(title)) {
 				if(tiddlerInfo) {
 					// If the tiddler is known on the server and has been modified locally then it needs to be saved to the server
-				if(this.wiki.getChangeCount(title) > tiddlerInfo.changeCount) {
+					if(self.wiki.getChangeCount(title) > tiddlerInfo.changeCount) {
 						return true;
 					}
 				} else {
@@ -222,14 +225,17 @@ Syncer.prototype.isDirty = function() {
 			}
 		}
 		// Check tiddlers that are known from the server but not currently in the store
-	titles = Object.keys(this.tiddlerInfo);
+		titles = Object.keys(self.tiddlerInfo);
 		for(index=0; index<titles.length; index++) {
-		if(!this.wiki.tiddlerExists(titles[index])) {
+			if(!self.wiki.tiddlerExists(titles[index])) {
 				// There must be a pending delete
 				return true;
 			}
 		}
 		return false;
+	}
+	var dirtyStatus = checkIsDirty();
+	return dirtyStatus;
 };
 
 /*
@@ -293,92 +299,16 @@ Syncer.prototype.getStatus = function(callback) {
 Synchronise from the server by reading the skinny tiddler list and queuing up loads for any tiddlers that we don't already have up to date
 */
 Syncer.prototype.syncFromServer = function() {
-	var self = this,
+	if(this.canSyncFromServer()) {
-		cancelNextSync = function() {
+		this.forceSyncFromServer = true;
-			if(self.pollTimerId) {
+		this.processTaskQueue();	
-				clearTimeout(self.pollTimerId);
-				self.pollTimerId = null;
-			}
-		},
-		triggerNextSync = function() {
-			self.pollTimerId = setTimeout(function() {
-				self.pollTimerId = null;
-				self.syncFromServer.call(self);
-			},self.pollTimerInterval);
-		},
-		syncSystemFromServer = (self.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes" ? true : false);
-	if(this.syncadaptor && this.syncadaptor.getUpdatedTiddlers) {
-		this.logger.log("Retrieving updated tiddler list");
-		cancelNextSync();
-		this.syncadaptor.getUpdatedTiddlers(self,function(err,updates) {
-			triggerNextSync();
-			if(err) {
-				self.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
-				return;
-			}
-			if(updates) {
-				$tw.utils.each(updates.modifications,function(title) {
-					self.titlesToBeLoaded[title] = true;
-				});
-				$tw.utils.each(updates.deletions,function(title) {
-					if(syncSystemFromServer || !self.wiki.isSystemTiddler(title)) {
-						delete self.tiddlerInfo[title];
-						self.logger.log("Deleting tiddler missing from server:",title);
-						self.wiki.deleteTiddler(title);
-					}
-				});
-				if(updates.modifications.length > 0 || updates.deletions.length > 0) {
-					self.processTaskQueue();
-				}				
-			}
-		});
-	} else if(this.syncadaptor && this.syncadaptor.getSkinnyTiddlers) {
-		this.logger.log("Retrieving skinny tiddler list");
-		cancelNextSync();
-		this.syncadaptor.getSkinnyTiddlers(function(err,tiddlers) {
-			triggerNextSync();
-			// Check for errors
-			if(err) {
-				self.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
-				return;
-			}
-			// Keep track of which tiddlers we already know about have been reported this time
-			var previousTitles = Object.keys(self.tiddlerInfo);
-			// Process each incoming tiddler
-			for(var t=0; t<tiddlers.length; t++) {
-				// Get the incoming tiddler fields, and the existing tiddler
-				var tiddlerFields = tiddlers[t],
-					incomingRevision = tiddlerFields.revision + "",
-					tiddler = self.wiki.tiddlerExists(tiddlerFields.title) && self.wiki.getTiddler(tiddlerFields.title),
-					tiddlerInfo = self.tiddlerInfo[tiddlerFields.title],
-					currRevision = tiddlerInfo ? tiddlerInfo.revision : null,
-					indexInPreviousTitles = previousTitles.indexOf(tiddlerFields.title);
-				if(indexInPreviousTitles !== -1) {
-					previousTitles.splice(indexInPreviousTitles,1);
-				}
-				// Ignore the incoming tiddler if it's the same as the revision we've already got
-				if(currRevision !== incomingRevision) {
-					// Only load the skinny version if we don't already have a fat version of the tiddler
-					if(!tiddler || tiddler.fields.text === undefined) {
-						self.storeTiddler(tiddlerFields);
-					}
-					// Do a full load of this tiddler
-					self.titlesToBeLoaded[tiddlerFields.title] = true;
-				}
-			}
-			// Delete any tiddlers that were previously reported but missing this time
-			$tw.utils.each(previousTitles,function(title) {
-				if(syncSystemFromServer || !self.wiki.isSystemTiddler(title)) {
-					delete self.tiddlerInfo[title];
-					self.logger.log("Deleting tiddler missing from server:",title);
-					self.wiki.deleteTiddler(title);
-				}
-			});
-			self.processTaskQueue();
-		});
 	}
 };
 
+Syncer.prototype.canSyncFromServer = function() {
+	return !!this.syncadaptor.getUpdatedTiddlers || !!this.syncadaptor.getSkinnyTiddlers;
+}
+
 /*
 Force load a tiddler from the server
 */
@@ -518,31 +448,39 @@ Syncer.prototype.processTaskQueue = function() {
 			this.updateDirtyStatus();
 			// And trigger a timeout if there is a pending task
 			if(task === true) {
-				this.triggerTimeout();				
+				this.triggerTimeout(this.taskTimerInterval);
+			} else if(this.canSyncFromServer()) {
+				this.triggerTimeout(this.pollTimerInterval);
 			}
 		}
 	} else {
 		this.updateDirtyStatus();
+		this.triggerTimeout(this.taskTimerInterval);
 	}
 };
 
 Syncer.prototype.triggerTimeout = function(interval) {
 	var self = this;
-	if(!this.taskTimerId) {
+	if(this.taskTimerId) {
+		clearTimeout(this.taskTimerId);
+	}
 	this.taskTimerId = setTimeout(function() {
 		self.taskTimerId = null;
 		self.processTaskQueue.call(self);
 	},interval || self.taskTimerInterval);
-	}
 };
 
 /*
-Choose the next sync task. We prioritise saves, then deletes, then loads from the server
+Choose the next sync task. We prioritise saves to the server, then getting updates from the server, then deletes to the server, then loads from the server
 
-Returns either a task object, null if there's no upcoming tasks, or the boolean true if there are pending tasks that aren't yet due
+Returns either:
+* a task object
+* the boolean true if there are pending sync tasks that aren't yet due
+* null if there's no pending sync tasks (just the next poll)
 */
 Syncer.prototype.chooseNextTask = function() {
-	var thresholdLastSaved = (new Date()) - this.throttleInterval,
+	var now = new Date(),
+		thresholdLastSaved = now - this.throttleInterval,
 		havePending = null;
 	// First we look for tiddlers that have been modified locally and need saving back to the server
 	var titles = this.getSyncedTiddlers();
@@ -563,7 +501,11 @@ Syncer.prototype.chooseNextTask = function() {
 			}
 		}
 	}
-	// Second, we check tiddlers that are known from the server but not currently in the store, and so need deleting on the server
+	// Second we check for an outstanding sync from server
+	if(this.forceSyncFromServer || (this.timestampLastSyncFromServer && (now.valueOf() >= (this.timestampLastSyncFromServer.valueOf() + this.pollTimerInterval)))) {
+		return new SyncFromServerTask(this);
+	}
+	// Third, we check tiddlers that are known from the server but not currently in the store, and so need deleting on the server
 	titles = Object.keys(this.tiddlerInfo);
 	for(index=0; index<titles.length; index++) {
 		title = titles[index];
@@ -573,13 +515,13 @@ Syncer.prototype.chooseNextTask = function() {
 			return new DeleteTiddlerTask(this,title);
 		}
 	}
-	// Check for tiddlers that need loading
+	// Finally, check for tiddlers that need loading
 	title = Object.keys(this.titlesToBeLoaded)[0];
 	if(title) {
 		delete this.titlesToBeLoaded[title];
 		return new LoadTiddlerTask(this,title);
 	}
-	// No tasks are ready
+	// No tasks are ready now, but might be in the future
 	return havePending;
 };
 
@@ -589,6 +531,10 @@ function SaveTiddlerTask(syncer,title) {
 	this.type = "save";
 }
 
+SaveTiddlerTask.prototype.toString = function() {
+	return "SAVE " + this.title;
+}
+
 SaveTiddlerTask.prototype.run = function(callback) {
 	var self = this,
 		changeCount = this.syncer.wiki.getChangeCount(this.title),
@@ -613,7 +559,6 @@ SaveTiddlerTask.prototype.run = function(callback) {
 			tiddlerInfo: self.syncer.tiddlerInfo[self.title]
 		});
 	} else {
-		this.syncer.logger.log(" Not Dispatching 'save' task:",this.title,"tiddler does not exist");
 		$tw.utils.nextTick(callback(null));
 	}
 };
@@ -624,6 +569,10 @@ function DeleteTiddlerTask(syncer,title) {
 	this.type = "delete";
 }
 
+DeleteTiddlerTask.prototype.toString = function() {
+	return "DELETE " + this.title;
+}
+
 DeleteTiddlerTask.prototype.run = function(callback) {
 	var self = this;
 	this.syncer.logger.log("Dispatching 'delete' task:",this.title);
@@ -647,6 +596,10 @@ function LoadTiddlerTask(syncer,title) {
 	this.type = "load";
 }
 
+LoadTiddlerTask.prototype.toString = function() {
+	return "LOAD " + this.title;
+}
+
 LoadTiddlerTask.prototype.run = function(callback) {
 	var self = this;
 	this.syncer.logger.log("Dispatching 'load' task:",this.title);
@@ -664,6 +617,91 @@ LoadTiddlerTask.prototype.run = function(callback) {
 	});
 };
 
+function SyncFromServerTask(syncer) {
+	this.syncer = syncer;
+	this.type = "syncfromserver";
+}
+
+SyncFromServerTask.prototype.toString = function() {
+	return "SYNCFROMSERVER";
+}
+
+SyncFromServerTask.prototype.run = function(callback) {
+	var self = this;
+	var syncSystemFromServer = (self.syncer.wiki.getTiddlerText("$:/config/SyncSystemTiddlersFromServer") === "yes" ? true : false);
+	var successCallback = function() {
+		self.syncer.forceSyncFromServer = false;
+		self.syncer.timestampLastSyncFromServer = new Date();
+		callback(null);
+	};
+	if(this.syncer.syncadaptor.getUpdatedTiddlers) {
+		this.syncer.syncadaptor.getUpdatedTiddlers(self.syncer,function(err,updates) {
+			if(err) {
+				self.syncer.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
+				return callback(err);
+			}
+			if(updates) {
+				$tw.utils.each(updates.modifications,function(title) {
+					self.syncer.titlesToBeLoaded[title] = true;
+				});
+				$tw.utils.each(updates.deletions,function(title) {
+					if(syncSystemFromServer || !self.syncer.wiki.isSystemTiddler(title)) {
+						delete self.syncer.tiddlerInfo[title];
+						self.syncer.logger.log("Deleting tiddler missing from server:",title);
+						self.syncer.wiki.deleteTiddler(title);
+					}
+				});
+			}
+			return successCallback();
+		});
+	} else if(this.syncer.syncadaptor.getSkinnyTiddlers) {
+		this.syncer.syncadaptor.getSkinnyTiddlers(function(err,tiddlers) {
+			// Check for errors
+			if(err) {
+				self.syncer.displayError($tw.language.getString("Error/RetrievingSkinny"),err);
+				return callback(err);
+			}
+			// Keep track of which tiddlers we already know about have been reported this time
+			var previousTitles = Object.keys(self.syncer.tiddlerInfo);
+			// Process each incoming tiddler
+			for(var t=0; t<tiddlers.length; t++) {
+				// Get the incoming tiddler fields, and the existing tiddler
+				var tiddlerFields = tiddlers[t],
+					incomingRevision = tiddlerFields.revision + "",
+					tiddler = self.syncer.wiki.tiddlerExists(tiddlerFields.title) && self.syncer.wiki.getTiddler(tiddlerFields.title),
+					tiddlerInfo = self.syncer.tiddlerInfo[tiddlerFields.title],
+					currRevision = tiddlerInfo ? tiddlerInfo.revision : null,
+					indexInPreviousTitles = previousTitles.indexOf(tiddlerFields.title);
+				if(indexInPreviousTitles !== -1) {
+					previousTitles.splice(indexInPreviousTitles,1);
+				}
+				// Ignore the incoming tiddler if it's the same as the revision we've already got
+				if(currRevision !== incomingRevision) {
+					// Only load the skinny version if we don't already have a fat version of the tiddler
+					if(!tiddler || tiddler.fields.text === undefined) {
+						self.syncer.storeTiddler(tiddlerFields);
+					}
+					// Do a full load of this tiddler
+					self.syncer.titlesToBeLoaded[tiddlerFields.title] = true;
+				}
+			}
+			// Delete any tiddlers that were previously reported but missing this time
+			$tw.utils.each(previousTitles,function(title) {
+				if(syncSystemFromServer || !self.syncer.wiki.isSystemTiddler(title)) {
+					delete self.syncer.tiddlerInfo[title];
+					self.syncer.logger.log("Deleting tiddler missing from server:",title);
+					self.syncer.wiki.deleteTiddler(title);
+				}
+			});
+			self.syncer.forceSyncFromServer = false;
+			self.syncer.timestampLastSyncFromServer = new Date();
+			return successCallback();
+		});
+	} else {
+		return successCallback();
+	}
+};
+
 exports.Syncer = Syncer;
 
 })();

core/modules/utils/dom/dom.js

@@ -292,7 +292,9 @@ exports.copyToClipboard = function(text,options) {
 	} catch (err) {
 	}
 	if(!options.doNotNotify) {
-		$tw.notifier.display(succeeded ? "$:/language/Notifications/CopiedToClipboard/Succeeded" : "$:/language/Notifications/CopiedToClipboard/Failed");
+		var successNotification = options.successNotification || "$:/language/Notifications/CopiedToClipboard/Succeeded",
+			failureNotification = options.failureNotification || "$:/language/Notifications/CopiedToClipboard/Failed"
+		$tw.notifier.display(succeeded ? successNotification : failureNotification);
 	}
 	document.body.removeChild(textArea);
 };
@@ -313,7 +315,7 @@ exports.collectDOMVariables = function(selectedNode,domNode,event) {
 			variables["dom-" + attribute.name] = attribute.value.toString();
 		});
 		
-		if(selectedNode.offsetLeft) {
+		if("offsetLeft" in selectedNode) {
 			// Add variables with a (relative and absolute) popup coordinate string for the selected node
 			var nodeRect = {
 				left: selectedNode.offsetLeft,
@@ -338,12 +340,12 @@ exports.collectDOMVariables = function(selectedNode,domNode,event) {
 		}
 	}
 	
-	if(domNode && domNode.offsetWidth) {
+	if(domNode && ("offsetWidth" in domNode)) {
 		variables["tv-widgetnode-width"] = domNode.offsetWidth.toString();
 		variables["tv-widgetnode-height"] = domNode.offsetHeight.toString();
 	}
 
-	if(event && event.clientX && event.clientY) {
+	if(event && ("clientX" in event) && ("clientY" in event)) {
 		if(selectedNode) {
 			// Add variables for event X and Y position relative to selected node
 			selectedNodeRect = selectedNode.getBoundingClientRect();

core/modules/utils/dom/http.js

@@ -100,6 +100,10 @@ headers: hashmap of header name to header value to be sent with the request
 passwordHeaders: hashmap of header name to password store name to be sent with the request
 queryStrings: hashmap of query string parameter name to parameter value to be sent with the request
 passwordQueryStrings: hashmap of query string parameter name to password store name to be sent with the request
+basicAuthUsername: plain username for basic authentication
+basicAuthUsernameFromStore: name of password store entry containing username
+basicAuthPassword: plain password for basic authentication
+basicAuthPasswordFromStore: name of password store entry containing password
 */
 function HttpClientRequest(options) {
 	var self = this;
@@ -112,6 +116,7 @@ function HttpClientRequest(options) {
 	this.method = options.method || "GET";
 	this.body = options.body || "";
 	this.binary = options.binary || "";
+	this.useDefaultHeaders = options.useDefaultHeaders !== "false" ? true : false,
 	this.variables = options.variables;
 	var url = options.url;
 	$tw.utils.each(options.queryStrings,function(value,name) {
@@ -128,6 +133,11 @@ function HttpClientRequest(options) {
 	$tw.utils.each(options.passwordHeaders,function(value,name) {
 		self.requestHeaders[name] = $tw.utils.getPassword(value) || "";
 	});
+	this.basicAuthUsername = options.basicAuthUsername || (options.basicAuthUsernameFromStore && $tw.utils.getPassword(options.basicAuthUsernameFromStore)) || "";
+	this.basicAuthPassword = options.basicAuthPassword || (options.basicAuthPasswordFromStore && $tw.utils.getPassword(options.basicAuthPasswordFromStore)) || "";
+	if(this.basicAuthUsername && this.basicAuthPassword) {
+		this.requestHeaders.Authorization = "Basic " + $tw.utils.base64Encode(this.basicAuthUsername + ":" + this.basicAuthPassword);
+	}
 }
 
 HttpClientRequest.prototype.send = function(callback) {
@@ -156,6 +166,7 @@ HttpClientRequest.prototype.send = function(callback) {
 		this.xhr = $tw.utils.httpRequest({
 			url: this.url,
 			type: this.method,
+			useDefaultHeaders: this.useDefaultHeaders,
 			headers: this.requestHeaders,
 			data: this.body,
 			returnProp: this.binary === "" ? "responseText" : "response",
@@ -187,7 +198,7 @@ HttpClientRequest.prototype.send = function(callback) {
 					for (var i=0; i<len; i++) {
 						binary += String.fromCharCode(bytes[i]);
 					}
-					resultVariables.data = window.btoa(binary);
+					resultVariables.data = $tw.utils.base64Encode(binary,true);
 				}
 				self.wiki.addTiddler(new $tw.Tiddler(self.wiki.getTiddler(requestTrackerTitle),{
 					status: completionCode,
@@ -231,7 +242,8 @@ Make an HTTP request. Options are:
 exports.httpRequest = function(options) {
 	var type = options.type || "GET",
 		url = options.url,
-		headers = options.headers || {accept: "application/json"},
+		useDefaultHeaders = options.useDefaultHeaders !== false ? true : false,
+		headers = options.headers || (useDefaultHeaders ? {accept: "application/json"} : {}),
 		hasHeader = function(targetHeader) {
 			targetHeader = targetHeader.toLowerCase();
 			var result = false;
@@ -283,13 +295,13 @@ exports.httpRequest = function(options) {
 	// Set up the state change handler
 	request.onreadystatechange = function() {
 		if(this.readyState === 4) {
-			if(this.status === 200 || this.status === 201 || this.status === 204) {
+			if(this.status >= 200 && this.status < 300) {
 				// Success!
 				options.callback(null,this[returnProp],this);
 				return;
 			}
 		// Something went wrong
-		options.callback($tw.language.getString("Error/XMLHttpRequest") + ": " + this.status,null,this);
+		options.callback($tw.language.getString("Error/XMLHttpRequest") + ": " + this.status,this[returnProp],this);
 		}
 	};
 	// Handle progress
@@ -307,10 +319,10 @@ exports.httpRequest = function(options) {
 			request.setRequestHeader(headerTitle,header);
 		});
 	}
-	if(data && !hasHeader("Content-Type")) {
+	if(data && !hasHeader("Content-Type") && useDefaultHeaders) {
 		request.setRequestHeader("Content-Type","application/x-www-form-urlencoded; charset=UTF-8");
 	}
-	if(!hasHeader("X-Requested-With") && !isSimpleRequest(type,headers)) {
+	if(!hasHeader("X-Requested-With") && !isSimpleRequest(type,headers) && useDefaultHeaders) {
 		request.setRequestHeader("X-Requested-With","TiddlyWiki");
 	}
 	// Send data

core/modules/utils/errors.js

@@ -0,0 +1,23 @@
+/*\
+title: $:/core/modules/utils/errors.js
+type: application/javascript
+module-type: utils
+
+Custom errors for TiddlyWiki.
+
+\*/
+(function(){
+
+function TranscludeRecursionError() {
+	Error.apply(this,arguments);
+	this.signatures = Object.create(null);
+};
+
+/* Maximum permitted depth of the widget tree for recursion detection */
+TranscludeRecursionError.MAX_WIDGET_TREE_DEPTH = 1000;
+
+TranscludeRecursionError.prototype = Object.create(Error);
+
+exports.TranscludeRecursionError = TranscludeRecursionError;
+
+})();

core/modules/utils/fakedom.js

@@ -42,7 +42,7 @@ var TW_TextNode = function(text) {
 	this.textContent = text + "";
 };
 
-Object.setPrototypeOf(TW_TextNode,TW_Node.prototype);
+Object.setPrototypeOf(TW_TextNode.prototype,TW_Node.prototype);
 
 Object.defineProperty(TW_TextNode.prototype, "nodeType", {
 	get: function() {
@@ -67,7 +67,7 @@ var TW_Element = function(tag,namespace) {
 	this.namespaceURI = namespace || "http://www.w3.org/1999/xhtml";
 };
 
-Object.setPrototypeOf(TW_Element,TW_Node.prototype);
+Object.setPrototypeOf(TW_Element.prototype,TW_Node.prototype);
 
 Object.defineProperty(TW_Element.prototype, "style", {
 	get: function() {
@@ -104,7 +104,11 @@ TW_Element.prototype.setAttribute = function(name,value) {
 	if(this.isRaw) {
 		throw "Cannot setAttribute on a raw TW_Element";
 	}
+	if(name === "style") {
+		this.style = value;
+	} else {
 		this.attributes[name] = value + "";
+	}
 };
 
 TW_Element.prototype.setAttributeNS = function(namespace,name,value) {

core/modules/utils/filesystem.js

@@ -316,11 +316,13 @@ Options include:
 	pathFilters: optional array of filters to be used to generate the base path
 	wiki: optional wiki for evaluating the pathFilters
 	fileInfo: an existing fileInfo object to check against
+	fileInfo.overwrite: if true, turns off filename clash numbers (defaults to false)
 */
 exports.generateTiddlerFilepath = function(title,options) {
 	var directory = options.directory || "",
 		extension = options.extension || "",
 		originalpath = (options.fileInfo && options.fileInfo.originalpath) ? options.fileInfo.originalpath : "",
+		overwrite = options.fileInfo && options.fileInfo.overwrite || false,
 		filepath;
 	// Check if any of the pathFilters applies
 	if(options.pathFilters && options.wiki) {
@@ -381,19 +383,20 @@ exports.generateTiddlerFilepath = function(title,options) {
 			filepath += char.charCodeAt(0).toString();
 		});
 	}
-	// Add a uniquifier if the file already exists
+	// Add a uniquifier if the file already exists (default)
-	var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined,
+	var fullPath = path.resolve(directory, filepath + extension);
+	if (!overwrite) {
+		var oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined,
 		count = 0;
 		do {
 			fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
-		if(oldPath && oldPath == fullPath) {
+			if(oldPath && oldPath == fullPath) break;
-			break;
-		}
 			count++;
 		} while(fs.existsSync(fullPath));
+	}
 	// If the last write failed with an error, or if path does not start with:
 	//	the resolved options.directory, the resolved wikiPath directory, the wikiTiddlersPath directory, 
-	//	or the 'originalpath' directory, then $tw.utils.encodeURIComponentExtended() and resolve to tiddler directory.
+	//	or the 'originalpath' directory, then $tw.utils.encodeURIComponentExtended() and resolve to options.directory.
 	var writePath = $tw.hooks.invokeHook("th-make-tiddler-path",fullPath,fullPath),
 		encode = (options.fileInfo || {writeError: false}).writeError == true;
 	if(!encode) {

core/modules/utils/repository.js

@@ -0,0 +1,52 @@
+/*\
+title: $:/core/modules/utils/repository.js
+type: application/javascript
+module-type: utils
+
+Utilities for working with the TiddlyWiki repository file structure
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Get an object containing all the plugins as a hashmap by title of the JSON representation of the plugin
+Options:
+
+ignoreEnvironmentVariables: defaults to false
+*/
+exports.getAllPlugins = function(options) {
+	options = options || {};
+	var fs = require("fs"),
+		path = require("path"),
+		tiddlers = {};
+	// Collect up the library plugins
+	var collectPlugins = function(folder) {
+			var pluginFolders = $tw.utils.getSubdirectories(folder) || [];
+			for(var p=0; p<pluginFolders.length; p++) {
+				if(!$tw.boot.excludeRegExp.test(pluginFolders[p])) {
+					var pluginFields = $tw.loadPluginFolder(path.resolve(folder,"./" + pluginFolders[p]));
+					if(pluginFields && pluginFields.title) {
+						tiddlers[pluginFields.title] = pluginFields;
+					}
+				}
+			}
+		},
+		collectPublisherPlugins = function(folder) {
+			var publisherFolders = $tw.utils.getSubdirectories(folder) || [];
+			for(var t=0; t<publisherFolders.length; t++) {
+				if(!$tw.boot.excludeRegExp.test(publisherFolders[t])) {
+					collectPlugins(path.resolve(folder,"./" + publisherFolders[t]));
+				}
+			}
+		};
+	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.pluginsPath,options.ignoreEnvironmentVariables ? undefined : $tw.config.pluginsEnvVar),collectPublisherPlugins);
+	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.themesPath,options.ignoreEnvironmentVariables ? undefined : $tw.config.themesEnvVar),collectPublisherPlugins);
+	$tw.utils.each($tw.getLibraryItemSearchPaths($tw.config.languagesPath,options.ignoreEnvironmentVariables ? undefined : $tw.config.languagesEnvVar),collectPlugins);
+	return tiddlers;
+};
+
+})();

core/modules/utils/utils.js

@@ -819,18 +819,50 @@ exports.hashString = function(str) {
 	},0);
 };
 
+/*
+Cryptographic hash function as used by sha256 filter operator
+options.length .. number of characters returned defaults to 64
+*/
+exports.sha256 = function(str, options) {
+	options = options || {}
+	return $tw.sjcl.codec.hex.fromBits($tw.sjcl.hash.sha256.hash(str)).substr(0,options.length || 64);
+}
+
+/*
+Base64 utility functions that work in either browser or Node.js
+*/
+if(typeof window !== 'undefined') {
+	exports.btoa = function(binstr) { return window.btoa(binstr); }
+	exports.atob = function(b64) { return window.atob(b64); }
+} else {
+	exports.btoa = function(binstr) {
+		return Buffer.from(binstr, 'binary').toString('base64');
+	}
+	exports.atob = function(b64) {
+		return Buffer.from(b64, 'base64').toString('binary');
+	}
+}
+
 /*
 Decode a base64 string
 */
-exports.base64Decode = function(string64) {
+exports.base64Decode = function(string64,binary,urlsafe) {
-	return base64utf8.base64.decode.call(base64utf8,string64);
+	var encoded = urlsafe ? string64.replace(/_/g,'/').replace(/-/g,'+') : string64;
+	if(binary) return exports.atob(encoded)
+	else return base64utf8.base64.decode.call(base64utf8,encoded);
 };
 
 /*
 Encode a string to base64
 */
-exports.base64Encode = function(string64) {
+exports.base64Encode = function(string64,binary,urlsafe) {
-	return base64utf8.base64.encode.call(base64utf8,string64);
+	var encoded;
+	if(binary) encoded = exports.btoa(string64);
+	else encoded = base64utf8.base64.encode.call(base64utf8,string64);
+	if(urlsafe) {
+		encoded = encoded.replace(/\+/g,'-').replace(/\//g,'_');
+	}
+	return encoded;
 };
 
 /*

core/modules/widgets/action-deletefield.js

@@ -37,6 +37,7 @@ Compute the internal state of the widget
 DeleteFieldWidget.prototype.execute = function() {
 	this.actionTiddler = this.getAttribute("$tiddler",this.getVariable("currentTiddler"));
 	this.actionField = this.getAttribute("$field",null);
+	this.actionTimestamp = this.getAttribute("$timestamp","yes") === "yes";
 };
 
 /*
@@ -69,11 +70,15 @@ DeleteFieldWidget.prototype.invokeAction = function(triggeringWidget,event) {
 		$tw.utils.each(this.attributes,function(attribute,name) {
 			if(name.charAt(0) !== "$" && name !== "title") {
 				removeFields[name] = undefined;
+				if(name in tiddler.fields) {
 					hasChanged = true;
 				}
+			}
 		});
 		if(hasChanged) {
-			this.wiki.addTiddler(new $tw.Tiddler(this.wiki.getCreationFields(),tiddler,removeFields,this.wiki.getModificationFields()));
+			var creationFields = this.actionTimestamp ? this.wiki.getCreationFields() : {};
+			var modificationFields = this.actionTimestamp ? this.wiki.getModificationFields() : {};
+			this.wiki.addTiddler(new $tw.Tiddler(creationFields,tiddler,removeFields,modificationFields));
 		}
 	}
 	return true; // Action was invoked

core/modules/widgets/action-log.js

@@ -66,8 +66,13 @@ LogWidget.prototype.log = function() {
 	});
 
 	for(var v in this.variables) {
+		var variable = this.parentWidget && this.parentWidget.variables[v];
+		if(variable && variable.isFunctionDefinition) {
+			allVars[v] = variable.value;
+		} else {
 			allVars[v]  = this.getVariable(v,{defaultValue:""});
 		}
+	}
 	if(this.filter) {
 		filteredVars = this.wiki.compileFilter(this.filter).call(this.wiki,this.wiki.makeTiddlerIterator(allVars));
 		$tw.utils.each(filteredVars,function(name) {

core/modules/widgets/browse.js

@@ -70,6 +70,11 @@ BrowseWidget.prototype.render = function(parent,nextSibling) {
 		}
 		return false;
 	},false);
+	// Assign data- attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Insert element
 	parent.insertBefore(domNode,nextSibling);
 	this.renderChildren(domNode,null);
@@ -95,6 +100,11 @@ BrowseWidget.prototype.execute = function() {
 Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
 */
 BrowseWidget.prototype.refresh = function(changedTiddlers) {
+	var changedAttributes = this.computeAttributes();
+	if($tw.utils.count(changedAttributes) > 0) {
+		this.refreshSelf();
+		return true;	
+	}
 	return false;
 };
 

core/modules/widgets/button.js

@@ -59,6 +59,11 @@ ButtonWidget.prototype.render = function(parent,nextSibling) {
 		$tw.utils.pushTop(classes,"tc-popup-handle");
 	}
 	domNode.className = classes.join(" ");
+	// Assign data- attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Assign other attributes
 	if(this.style) {
 		domNode.setAttribute("style",this.style);
@@ -250,7 +255,7 @@ ButtonWidget.prototype.updateDomNodeClasses = function() {
 	//Add new classes from updated class attribute.
 	$tw.utils.pushTop(domNodeClasses,newClasses);
 	this.domNode.className = domNodeClasses.join(" ");
-}
+};
 
 /*
 Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
@@ -260,9 +265,16 @@ ButtonWidget.prototype.refresh = function(changedTiddlers) {
 	if(changedAttributes.actions || changedAttributes.to || changedAttributes.message || changedAttributes.param || changedAttributes.set || changedAttributes.setTo || changedAttributes.popup || changedAttributes.hover || changedAttributes.selectedClass || changedAttributes.style || changedAttributes.dragFilter || changedAttributes.dragTiddler || (this.set && changedTiddlers[this.set]) || (this.popup && changedTiddlers[this.popup]) || (this.popupTitle && changedTiddlers[this.popupTitle]) || changedAttributes.popupAbsCoords || changedAttributes.setTitle || changedAttributes.setField || changedAttributes.setIndex || changedAttributes.popupTitle || changedAttributes.disabled || changedAttributes["default"]) {
 		this.refreshSelf();
 		return true;
-	} else if(changedAttributes["class"]) {
+	} else {
+		if(changedAttributes["class"]) {
 			this.updateDomNodeClasses();
 		}
+		this.assignAttributes(this.domNodes[0],{
+			changedAttributes: changedAttributes,
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
+	}
 	return this.refreshChildren(changedTiddlers);
 };
 

core/modules/widgets/checkbox.js

@@ -53,6 +53,11 @@ CheckboxWidget.prototype.render = function(parent,nextSibling) {
 	this.labelDomNode.appendChild(this.inputDomNode);
 	this.spanDomNode = this.document.createElement("span");
 	this.labelDomNode.appendChild(this.spanDomNode);
+	// Assign data- attributes
+	this.assignAttributes(this.inputDomNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Add a click event handler
 	$tw.utils.addEventListeners(this.inputDomNode,[
 		{name: "change", handlerObject: this, handlerMethod: "handleChangeEvent"}
@@ -325,6 +330,11 @@ CheckboxWidget.prototype.refresh = function(changedTiddlers) {
 				$tw.utils.removeClass(this.labelDomNode,"tc-checkbox-checked");
 			}
 		}
+		this.assignAttributes(this.inputDomNode,{
+			changedAttributes: changedAttributes,
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
 		return this.refreshChildren(changedTiddlers) || refreshed;
 	}
 };
@@ -332,3 +342,4 @@ CheckboxWidget.prototype.refresh = function(changedTiddlers) {
 exports.checkbox = CheckboxWidget;
 
 })();
+	

core/modules/widgets/data.js

@@ -0,0 +1,182 @@
+/*\
+title: $:/core/modules/widgets/data.js
+type: application/javascript
+module-type: widget
+
+Widget to dynamically represent one or more tiddlers
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+var Widget = require("$:/core/modules/widgets/widget.js").widget;
+
+var DataWidget = function(parseTreeNode,options) {
+	this.dataWidgetTag = parseTreeNode.type;
+	this.initialise(parseTreeNode,options);
+};
+
+/*
+Inherit from the base widget class
+*/
+DataWidget.prototype = new Widget();
+
+/*
+Render this widget into the DOM
+*/
+DataWidget.prototype.render = function(parent,nextSibling) {
+	this.parentDomNode = parent;
+	this.computeAttributes();
+	this.execute();
+	this.dataPayload = this.computeDataTiddlerValues(); // Array of $tw.Tiddler objects
+	this.domNode = this.document.createTextNode(this.readDataTiddlerValuesAsJson());
+	parent.insertBefore(this.domNode,nextSibling);
+	this.domNodes.push(this.domNode);
+};
+
+/*
+Compute the internal state of the widget
+*/
+DataWidget.prototype.execute = function() {
+	// Nothing to do here
+};
+
+/*
+Read the tiddler value(s) from a data widget as an array of tiddler field objects (not $tw.Tiddler objects)
+*/
+DataWidget.prototype.readDataTiddlerValues = function() {
+	var results = [];
+	$tw.utils.each(this.dataPayload,function(tiddler,index) {
+		results.push(tiddler.getFieldStrings());
+	});
+	return results;
+};
+
+/*
+Read the tiddler value(s) from a data widget as an array of tiddler field objects (not $tw.Tiddler objects)
+*/
+DataWidget.prototype.readDataTiddlerValuesAsJson = function() {
+	return JSON.stringify(this.readDataTiddlerValues(),null,4);
+};
+
+/*
+Compute list of tiddlers from a data widget
+*/
+DataWidget.prototype.computeDataTiddlerValues = function() {
+	var self = this;
+	// Read any attributes not prefixed with $
+	var item = Object.create(null);
+	$tw.utils.each(this.attributes,function(value,name) {
+		if(name.charAt(0) !== "$") {
+			item[name] = value;	
+		}
+	});
+	// Deal with $tiddler, $filter or $compound-tiddler attributes
+	var tiddlers = [],title;
+	if(this.hasAttribute("$tiddler")) {
+		title = this.getAttribute("$tiddler");
+		if(title) {
+			var tiddler = this.wiki.getTiddler(title);
+			if(tiddler) {
+				tiddlers.push(tiddler);
+			}
+		}
+	}
+	if(this.hasAttribute("$filter")) {
+		var filter = this.getAttribute("$filter");
+		if(filter) {
+			var titles = this.wiki.filterTiddlers(filter);
+			$tw.utils.each(titles,function(title) {
+				var tiddler = self.wiki.getTiddler(title);
+				tiddlers.push(tiddler);
+			});
+		}
+	}
+	if(this.hasAttribute("$compound-tiddler")) {
+		title = this.getAttribute("$compound-tiddler");
+		if(title) {
+			tiddlers.push.apply(tiddlers,this.extractCompoundTiddler(title));
+		}
+	}
+	// Return the literal item if none of the special attributes were used
+	if(!this.hasAttribute("$tiddler") && !this.hasAttribute("$filter") && !this.hasAttribute("$compound-tiddler")) {
+		if(Object.keys(item).length > 0 && !!item.title) {
+			return [new $tw.Tiddler(item)];
+		} else {
+			return [];
+		}
+	} else {
+		// Apply the item fields to each of the tiddlers
+		delete item.title; // Do not overwrite the title
+		if(Object.keys(item).length > 0) {
+			$tw.utils.each(tiddlers,function(tiddler,index) {
+				tiddlers[index] = new $tw.Tiddler(tiddler,item);
+			});
+		}
+		return tiddlers;
+	}
+};
+
+/*
+Helper to extract tiddlers from text/vnd.tiddlywiki-multiple tiddlers
+*/
+DataWidget.prototype.extractCompoundTiddler = function(title) {
+	var tiddler = this.wiki.getTiddler(title);
+	if(tiddler && tiddler.fields.type === "text/vnd.tiddlywiki-multiple") {
+		var text = tiddler.fields.text || "",
+			rawTiddlers = text.split(/\r?\n\+\r?\n/),
+			tiddlers = [];
+		$tw.utils.each(rawTiddlers,function(rawTiddler) {
+			var fields = Object.create(null),
+				split = rawTiddler.split(/\r?\n\r?\n/mg);
+			if(split.length >= 1) {
+				fields = $tw.utils.parseFields(split[0],fields);
+			}
+			if(split.length >= 2) {
+				fields.text = split.slice(1).join("\n\n");
+			}
+			tiddlers.push(new $tw.Tiddler(fields));
+		});
+		return tiddlers;
+	} else {
+		return [];
+	}
+};
+
+/*
+Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
+*/
+DataWidget.prototype.refresh = function(changedTiddlers) {
+	var changedAttributes = this.computeAttributes();
+	var newPayload = this.computeDataTiddlerValues();
+	if(hasPayloadChanged(this.dataPayload,newPayload)) {
+		this.dataPayload = newPayload;
+		this.domNode.textContent = this.readDataTiddlerValuesAsJson();
+		return true;
+	} else {
+		return false;
+	}
+};
+
+/*
+Compare two arrays of tiddlers and return true if they are different
+*/
+function hasPayloadChanged(a,b) {
+	if(a.length === b.length) {
+		for(var t=0; t<a.length; t++) {
+			if(!(a[t].isEqual(b[t]))) {
+				return true;
+			}
+		}
+		return false;
+	} else {
+		return true;
+	}
+}
+
+exports.data = DataWidget;
+
+})();

core/modules/widgets/draggable.js

@@ -52,6 +52,11 @@ DraggableWidget.prototype.render = function(parent,nextSibling) {
 		classes.push("tc-draggable");
 	}
 	domNode.setAttribute("class",classes.join(" "));
+	// Assign data- attributes and style. attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Insert the node into the DOM and render any children
 	parent.insertBefore(domNode,nextSibling);
 	this.renderChildren(domNode,null);
@@ -108,13 +113,19 @@ DraggableWidget.prototype.updateDomNodeClasses = function() {
 Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
 */
 DraggableWidget.prototype.refresh = function(changedTiddlers) {
-	var changedAttributes = this.computeAttributes(),
+	var changedAttributes = this.computeAttributes();
-		changedAttributesCount = $tw.utils.count(changedAttributes);
+	if(changedAttributes.tag || changedAttributes.selector || changedAttributes.dragimagetype || changedAttributes.enable || changedAttributes.startactions || changedAttributes.endactions) {
-	if(changedAttributesCount === 1 && changedAttributes["class"]) {
-		this.updateDomNodeClasses();
-	} else if(changedAttributesCount > 0) {
 		this.refreshSelf();
 		return true;
+	} else {
+		if(changedAttributes["class"]) {
+			this.updateDomNodeClasses();
+		}
+		this.assignAttributes(this.domNodes[0],{
+			changedAttributes: changedAttributes,
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
 	}
 	return this.refreshChildren(changedTiddlers);
 };

core/modules/widgets/droppable.js

@@ -42,6 +42,11 @@ DroppableWidget.prototype.render = function(parent,nextSibling) {
 	domNode = this.document.createElement(tag);
 	this.domNode = domNode;
 	this.assignDomNodeClasses();
+	// Assign data- attributes and style. attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Add event handlers
 	if(this.droppableEnable) {
 		$tw.utils.addEventListeners(domNode,[
@@ -166,9 +171,16 @@ DroppableWidget.prototype.refresh = function(changedTiddlers) {
 	if(changedAttributes.tag || changedAttributes.enable || changedAttributes.disabledClass || changedAttributes.actions || changedAttributes.effect) {
 		this.refreshSelf();
 		return true;
-	} else if(changedAttributes["class"]) {
+	} else {
+		if(changedAttributes["class"]) {
 			this.assignDomNodeClasses();
 		}
+		this.assignAttributes(this.domNodes[0],{
+			changedAttributes: changedAttributes,
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
+	}
 	return this.refreshChildren(changedTiddlers);
 };
 

core/modules/widgets/edit.js

@@ -90,7 +90,7 @@ Selectively refreshes the widget if needed. Returns true if the widget or any of
 EditWidget.prototype.refresh = function(changedTiddlers) {
 	var changedAttributes = this.computeAttributes();
 	// Refresh if an attribute has changed, or the type associated with the target tiddler has changed
-	if(changedAttributes.tiddler || changedAttributes.field || changedAttributes.index || changedAttributes.tabindex || changedAttributes.cancelPopups || changedAttributes.inputActions || changedAttributes.refreshTitle || changedAttributes.autocomplete || (changedTiddlers[this.editTitle] && this.getEditorType() !== this.editorType)) {
+	if(changedAttributes.tiddler || changedAttributes.field || changedAttributes.index || changedAttributes.tabindex || changedAttributes.cancelPopups || changedAttributes.inputActions || changedAttributes.refreshTitle || changedAttributes.autocomplete || (this.getEditorType() !== this.editorType)) {
 		this.refreshSelf();
 		return true;
 	} else {

core/modules/widgets/image.js

@@ -58,24 +58,25 @@ ImageWidget.prototype.render = function(parent,nextSibling) {
 		if(this.wiki.isImageTiddler(this.imageSource)) {
 			var type = tiddler.fields.type,
 				text = tiddler.fields.text,
-				_canonical_uri = tiddler.fields._canonical_uri;
+				_canonical_uri = tiddler.fields._canonical_uri,
+				typeInfo = $tw.config.contentTypeInfo[type] || {},
+				deserializerType = typeInfo.deserializerType || type;
 			// If the tiddler has body text then it doesn't need to be lazily loaded
 			if(text) {
-				// Render the appropriate element for the image type
+				// Render the appropriate element for the image type by looking up the encoding in the content type info
-				switch(type) {
+				var encoding = typeInfo.encoding || "utf8";
-					case "application/pdf":
+				if (encoding === "base64") {
+					// .pdf .png .jpg etc.
+					src = "data:" + deserializerType + ";base64," + text;
+					if (deserializerType === "application/pdf") {
 						tag = "embed";
-						src = "data:application/pdf;base64," + text;
+					}
-						break;
+				} else {
-					case "image/svg+xml":
+					// .svg .tid .xml etc.
-						src = "data:image/svg+xml," + encodeURIComponent(text);
+					src = "data:" + deserializerType + "," + encodeURIComponent(text);
-						break;
-					default:
-						src = "data:" + type + ";base64," + text;
-						break;
 				}
 			} else if(_canonical_uri) {
-				switch(type) {
+				switch(deserializerType) {
 					case "application/pdf":
 						tag = "embed";
 						src = _canonical_uri;
@@ -99,6 +100,9 @@ ImageWidget.prototype.render = function(parent,nextSibling) {
 	if(this.imageClass) {
 		domNode.setAttribute("class",this.imageClass);
 	}
+	if(this.imageUsemap) {
+	    	domNode.setAttribute("usemap",this.imageUsemap);
+	}
 	if(this.imageWidth) {
 		domNode.setAttribute("width",this.imageWidth);
 	}
@@ -138,6 +142,7 @@ ImageWidget.prototype.execute = function() {
 	this.imageWidth = this.getAttribute("width");
 	this.imageHeight = this.getAttribute("height");
 	this.imageClass = this.getAttribute("class");
+    	this.imageUsemap = this.getAttribute("usemap");
 	this.imageTooltip = this.getAttribute("tooltip");
 	this.imageAlt = this.getAttribute("alt");
 	this.lazyLoading = this.getAttribute("loading");
@@ -148,7 +153,7 @@ Selectively refreshes the widget if needed. Returns true if the widget or any of
 */
 ImageWidget.prototype.refresh = function(changedTiddlers) {
 	var changedAttributes = this.computeAttributes();
-	if(changedAttributes.source || changedAttributes.width || changedAttributes.height || changedAttributes["class"] || changedAttributes.tooltip || changedTiddlers[this.imageSource]) {
+	if(changedAttributes.source || changedAttributes.width || changedAttributes.height || changedAttributes["class"] || changedAttributes.usemap || changedAttributes.tooltip || changedTiddlers[this.imageSource]) {
 		this.refreshSelf();
 		return true;
 	} else {

core/modules/widgets/importvariables.js

@@ -49,7 +49,7 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 	this.tiddlerList = tiddlerList || this.wiki.filterTiddlers(this.filter,this);
 	// Accumulate the <$set> widgets from each tiddler
 	$tw.utils.each(this.tiddlerList,function(title) {
-		var parser = widgetPointer.wiki.parseTiddler(title,{parseAsInline:true, configTrimWhiteSpace:true});
+		var parser = widgetPointer.wiki.parseTiddler(title,{parseAsInline:true, configTrimWhiteSpace:false});
 		if(parser) {
 			var parseTreeNode = parser.tree[0];
 			while(parseTreeNode && ["setvariable","set","parameters"].indexOf(parseTreeNode.type) !== -1) {

core/modules/widgets/link.js

@@ -43,6 +43,11 @@ LinkWidget.prototype.render = function(parent,nextSibling) {
 	} else {
 		// Just insert the link text
 		var domNode = this.document.createElement("span");
+		// Assign data- attributes
+		this.assignAttributes(domNode,{
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
 		parent.insertBefore(domNode,nextSibling);
 		this.renderChildren(domNode,null);
 		this.domNodes.push(domNode);
@@ -138,6 +143,11 @@ LinkWidget.prototype.renderLink = function(parent,nextSibling) {
 			widget: this
 		});
 	}
+	// Assign data- attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Insert the link into the DOM and render any children
 	parent.insertBefore(domNode,nextSibling);
 	this.renderChildren(domNode,null);
@@ -209,8 +219,7 @@ Selectively refreshes the widget if needed. Returns true if the widget or any of
 */
 LinkWidget.prototype.refresh = function(changedTiddlers) {
 	var changedAttributes = this.computeAttributes();
-	if(changedAttributes.to || changedTiddlers[this.to] || changedAttributes["aria-label"] || changedAttributes.tooltip ||
+	if($tw.utils.count(changedAttributes) > 0 || changedTiddlers[this.to]) {
-		changedAttributes["class"] || changedAttributes.tabindex || changedAttributes.draggable || changedAttributes.tag) {
 		this.refreshSelf();
 		return true;
 	}
@@ -220,3 +229,4 @@ LinkWidget.prototype.refresh = function(changedTiddlers) {
 exports.link = LinkWidget;
 
 })();
+	

core/modules/widgets/list.js

@@ -28,6 +28,18 @@ Inherit from the base widget class
 */
 ListWidget.prototype = new Widget();
 
+ListWidget.prototype.initialise = function(parseTreeNode,options) {
+	// Bail if parseTreeNode is undefined, meaning that the ListWidget constructor was called without any arguments so that it can be subclassed
+	if(parseTreeNode === undefined) {
+		return;
+	}
+	// First call parent constructor to set everything else up
+	Widget.prototype.initialise.call(this,parseTreeNode,options);
+	// Now look for <$list-template> and <$list-empty> widgets as immediate child widgets
+	// This is safe to do during initialization because parse trees never change after creation
+	this.findExplicitTemplates();
+}
+
 /*
 Render this widget into the DOM
 */
@@ -38,8 +50,8 @@ ListWidget.prototype.render = function(parent,nextSibling) {
 		$tw.modules.applyMethods("storyview",this.storyViews);
 	}
 	this.parentDomNode = parent;
-	this.computeAttributes();
+	var changedAttributes = this.computeAttributes();
-	this.execute();
+	this.execute(changedAttributes);
 	this.renderChildren(parent,nextSibling);
 	// Construct the storyview
 	var StoryView = this.storyViews[this.storyViewName];
@@ -59,7 +71,8 @@ ListWidget.prototype.render = function(parent,nextSibling) {
 /*
 Compute the internal state of the widget
 */
-ListWidget.prototype.execute = function() {
+ListWidget.prototype.execute = function(changedAttributes) {
+	var self = this;
 	// Get our attributes
 	this.template = this.getAttribute("template");
 	this.editTemplate = this.getAttribute("editTemplate");
@@ -67,6 +80,10 @@ ListWidget.prototype.execute = function() {
 	this.counterName = this.getAttribute("counter");
 	this.storyViewName = this.getAttribute("storyview");
 	this.historyTitle = this.getAttribute("history");
+	// Create join template only if needed
+	if(this.join === undefined || (changedAttributes && changedAttributes.join)) {
+		this.join = this.makeJoinTemplate();
+	}
 	// Compose the list elements
 	this.list = this.getTiddlerList();
 	var members = [],
@@ -85,19 +102,58 @@ ListWidget.prototype.execute = function() {
 	this.history = [];
 };
 
+ListWidget.prototype.findExplicitTemplates = function() {
+	var self = this;
+	this.explicitListTemplate = null;
+	this.explicitEmptyTemplate = null;
+	this.explicitJoinTemplate = null;
+	this.hasTemplateInBody = false;
+	var searchChildren = function(childNodes) {
+		var foundInlineTemplate = false;
+		$tw.utils.each(childNodes,function(node) {
+			if(node.type === "list-template") {
+				self.explicitListTemplate = node.children;
+			} else if(node.type === "list-empty") {
+				self.explicitEmptyTemplate = node.children;
+			} else if(node.type === "list-join") {
+				self.explicitJoinTemplate = node.children;
+			} else if(node.type === "element" && node.tag === "p") {
+				searchChildren(node.children);
+				foundInlineTemplate = true;
+			} else {
+				foundInlineTemplate = true;
+			}
+		});
+		return foundInlineTemplate;
+	};
+	this.hasTemplateInBody = searchChildren(this.parseTreeNode.children);
+}
+
 ListWidget.prototype.getTiddlerList = function() {
+	var limit = $tw.utils.getInt(this.getAttribute("limit",""),undefined);
 	var defaultFilter = "[!is[system]sort[title]]";
-	return this.wiki.filterTiddlers(this.getAttribute("filter",defaultFilter),this);
+	var results = this.wiki.filterTiddlers(this.getAttribute("filter",defaultFilter),this);
+	if(limit !== undefined) {
+		if(limit >= 0) {
+			results = results.slice(0,limit);
+		} else {
+			results = results.slice(limit);
+		}
+	}
+	return results;
 };
 
 ListWidget.prototype.getEmptyMessage = function() {
 	var parser,
-		emptyMessage = this.getAttribute("emptyMessage","");
+		emptyMessage = this.getAttribute("emptyMessage");
-	// this.wiki.parseText() calls 
+	// If emptyMessage attribute is not present or empty then look for an explicit empty template
-	// new Parser(..), which should only be done, if needed, because it's heavy!
+	if(!emptyMessage) {
-	if (emptyMessage === "") {
+		if(this.explicitEmptyTemplate) {
+			return this.explicitEmptyTemplate;
+		} else {
 			return [];
 		}
+	}
 	parser = this.wiki.parseText("text/vnd.tiddlywiki",emptyMessage,{parseAsInline: true});
 	if(parser) {
 		return parser.tree;
@@ -106,6 +162,24 @@ ListWidget.prototype.getEmptyMessage = function() {
 	}
 };
 
+/*
+Compose the template for a join between list items
+*/
+ListWidget.prototype.makeJoinTemplate = function() {
+	var parser,
+		join = this.getAttribute("join","");
+	if(join) {
+		parser = this.wiki.parseText("text/vnd.tiddlywiki",join,{parseAsInline:true})
+		if(parser) {
+			return parser.tree;
+		} else {
+			return [];
+		}
+	} else {
+		return this.explicitJoinTemplate; // May be null, and that's fine
+	}
+};
+
 /*
 Compose the template for a list item
 */
@@ -114,6 +188,7 @@ ListWidget.prototype.makeItemTemplate = function(title,index) {
 	var tiddler = this.wiki.getTiddler(title),
 		isDraft = tiddler && tiddler.hasField("draft.of"),
 		template = this.template,
+		join = this.join,
 		templateTree;
 	if(isDraft && this.editTemplate) {
 		template = this.editTemplate;
@@ -122,9 +197,16 @@ ListWidget.prototype.makeItemTemplate = function(title,index) {
 	if(template) {
 		templateTree = [{type: "transclude", attributes: {tiddler: {type: "string", value: template}}}];
 	} else {
+		// Check for child nodes of the list widget
 		if(this.parseTreeNode.children && this.parseTreeNode.children.length > 0) {
+			// Check for a <$list-item> widget
+			if(this.explicitListTemplate) {
+				templateTree = this.explicitListTemplate;
+			} else if(this.hasTemplateInBody) {
 				templateTree = this.parseTreeNode.children;
-		} else {
+			}
+		}
+		if(!templateTree || templateTree.length === 0) {
 			// Default template is a link to the title
 			templateTree = [{type: "element", tag: this.parseTreeNode.isBlock ? "div" : "span", children: [{type: "link", attributes: {to: {type: "string", value: title}}, children: [
 				{type: "text", text: title}
@@ -132,12 +214,12 @@ ListWidget.prototype.makeItemTemplate = function(title,index) {
 		}
 	}
 	// Return the list item
-	var parseTreeNode = {type: "listitem", itemTitle: title, variableName: this.variableName, children: templateTree};
+	var parseTreeNode = {type: "listitem", itemTitle: title, variableName: this.variableName, children: templateTree, join: join};
+	parseTreeNode.isLast = index === this.list.length - 1;
 	if(this.counterName) {
 		parseTreeNode.counter = (index + 1).toString();
 		parseTreeNode.counterName = this.counterName;
 		parseTreeNode.isFirst = index === 0;
-		parseTreeNode.isLast = index === this.list.length - 1;
 	}
 	return parseTreeNode;
 };
@@ -153,7 +235,7 @@ ListWidget.prototype.refresh = function(changedTiddlers) {
 		this.storyview.refreshStart(changedTiddlers,changedAttributes);
 	}
 	// Completely refresh if any of our attributes have changed
-	if(changedAttributes.filter || changedAttributes.variable || changedAttributes.counter || changedAttributes.template || changedAttributes.editTemplate || changedAttributes.emptyMessage || changedAttributes.storyview || changedAttributes.history) {
+	if(changedAttributes.filter || changedAttributes.variable || changedAttributes.counter || changedAttributes.template || changedAttributes.editTemplate || changedAttributes.join || changedAttributes.emptyMessage || changedAttributes.storyview || changedAttributes.history) {
 		this.refreshSelf();
 		result = true;
 	} else {
@@ -225,6 +307,8 @@ ListWidget.prototype.handleListChanges = function(changedTiddlers) {
 		// If we are providing an counter variable then we must refresh the items, otherwise we can rearrange them
 		var hasRefreshed = false,t;
 		if(this.counterName) {
+			var mustRefreshOldLast = false;
+			var oldLength = this.children.length;
 			// Cycle through the list and remove and re-insert the first item that has changed, and all the remaining items
 			for(t=0; t<this.list.length; t++) {
 				if(hasRefreshed || !this.children[t] || this.children[t].parseTreeNode.itemTitle !== this.list[t]) {
@@ -232,6 +316,9 @@ ListWidget.prototype.handleListChanges = function(changedTiddlers) {
 						this.removeListItem(t);
 					}
 					this.insertListItem(t,this.list[t]);
+					if(!hasRefreshed && t === oldLength) {
+						mustRefreshOldLast = true;
+					}
 					hasRefreshed = true;
 				} else {
 					// Refresh the item we're reusing
@@ -239,6 +326,12 @@ ListWidget.prototype.handleListChanges = function(changedTiddlers) {
 					hasRefreshed = hasRefreshed || refreshed;
 				}
 			}
+			// If items were inserted then we must recreate the item that used to be at the last position as it is no longer last
+			if(mustRefreshOldLast && oldLength > 0) {
+				var oldLastIdx = oldLength-1;
+				this.removeListItem(oldLastIdx);
+				this.insertListItem(oldLastIdx,this.list[oldLastIdx]);
+			}
 			// If there are items to remove and we have not refreshed then recreate the item that will now be at the last position
 			if(!hasRefreshed && this.children.length > this.list.length) {
 				this.removeListItem(this.list.length-1);
@@ -246,10 +339,29 @@ ListWidget.prototype.handleListChanges = function(changedTiddlers) {
 			}
 		} else {
 			// Cycle through the list, inserting and removing list items as needed
+			var mustRecreateLastItem = false;
+			if(this.join && this.join.length) {
+				if(this.children.length !== this.list.length) {
+						mustRecreateLastItem = true;
+				} else if(prevList[prevList.length-1] !== this.list[this.list.length-1]) {
+						mustRecreateLastItem = true;
+				}
+			}
+			var isLast = false, wasLast = false;
 			for(t=0; t<this.list.length; t++) {
+				isLast = t === this.list.length-1;
 				var index = this.findListItem(t,this.list[t]);
+				wasLast = index === this.children.length-1;
+				if(wasLast && (index !== t || this.children.length !== this.list.length)) {
+					mustRecreateLastItem = !!(this.join && this.join.length);
+				}
 				if(index === undefined) {
 					// The list item must be inserted
+					if(isLast && mustRecreateLastItem && t>0) {
+						// First re-create previosly-last item that will no longer be last
+						this.removeListItem(t-1);
+						this.insertListItem(t-1,this.list[t-1]);
+					}
 					this.insertListItem(t,this.list[t]);
 					hasRefreshed = true;
 				} else {
@@ -258,12 +370,18 @@ ListWidget.prototype.handleListChanges = function(changedTiddlers) {
 						this.removeListItem(n);
 						hasRefreshed = true;
 					}
-					// Refresh the item we're reusing
+					// Refresh the item we're reusing, or recreate if necessary
+					if(mustRecreateLastItem && (isLast || wasLast)) {
+						this.removeListItem(t);
+						this.insertListItem(t,this.list[t]);
+						hasRefreshed = true;
+					} else {
 						var refreshed = this.children[t].refresh(changedTiddlers);
 						hasRefreshed = hasRefreshed || refreshed;
 					}
 				}
 			}
+		}
 		// Remove any left over items
 		for(t=this.children.length-1; t>=this.list.length; t--) {
 			this.removeListItem(t);
@@ -350,8 +468,17 @@ ListItemWidget.prototype.execute = function() {
 		this.setVariable(this.parseTreeNode.counterName + "-first",this.parseTreeNode.isFirst ? "yes" : "no");
 		this.setVariable(this.parseTreeNode.counterName + "-last",this.parseTreeNode.isLast ? "yes" : "no");
 	}
+	// Add join if needed
+	var children = this.parseTreeNode.children,
+		join = this.parseTreeNode.join;
+	if(join && join.length && !this.parseTreeNode.isLast) {
+		children = children.slice(0);
+		$tw.utils.each(join,function(joinNode) {
+			children.push(joinNode);
+		})
+	}
 	// Construct the child widgets
-	this.makeChildWidgets();
+	this.makeChildWidgets(children);
 };
 
 /*
@@ -363,4 +490,37 @@ ListItemWidget.prototype.refresh = function(changedTiddlers) {
 
 exports.listitem = ListItemWidget;
 
+/*
+Make <$list-template> and <$list-empty> widgets that do nothing
+*/
+var ListTemplateWidget = function(parseTreeNode,options) {
+	// Main initialisation inherited from widget.js
+	this.initialise(parseTreeNode,options);
+};
+ListTemplateWidget.prototype = new Widget();
+ListTemplateWidget.prototype.render = function() {}
+ListTemplateWidget.prototype.refresh = function() { return false; }
+
+exports["list-template"] = ListTemplateWidget;
+
+var ListEmptyWidget = function(parseTreeNode,options) {
+	// Main initialisation inherited from widget.js
+	this.initialise(parseTreeNode,options);
+};
+ListEmptyWidget.prototype = new Widget();
+ListEmptyWidget.prototype.render = function() {}
+ListEmptyWidget.prototype.refresh = function() { return false; }
+
+exports["list-empty"] = ListEmptyWidget;
+
+var ListJoinWidget = function(parseTreeNode,options) {
+	// Main initialisation inherited from widget.js
+	this.initialise(parseTreeNode,options);
+};
+ListJoinWidget.prototype = new Widget();
+ListJoinWidget.prototype.render = function() {}
+ListJoinWidget.prototype.refresh = function() { return false; }
+
+exports["list-join"] = ListJoinWidget;
+
 })();

core/modules/widgets/parameters.js

@@ -74,6 +74,18 @@ ParametersWidget.prototype.execute = function() {
 				self.setVariable(variableName,getValue(name));
 			}
 		});
+	} else {
+		// There is no parent transclude. i.e. direct rendering.
+		// We use default values only.
+		$tw.utils.each($tw.utils.getOrderedAttributesFromParseTreeNode(self.parseTreeNode),function(attr,index) {
+			var name = attr.name;
+			// If the attribute name starts with $$ then reduce to a single dollar
+			if(name.substr(0,2) === "$$") {
+				name = name.substr(1);
+			}
+			var value = self.getAttribute(attr.name,"");
+			self.setVariable(name,value);
+		});
 	}
 	// Construct the child widgets
 	this.makeChildWidgets();

core/modules/widgets/radio.js

@@ -40,6 +40,10 @@ RadioWidget.prototype.render = function(parent,nextSibling) {
 	);
 	this.inputDomNode = this.document.createElement("input");
 	this.inputDomNode.setAttribute("type","radio");
+	this.assignAttributes(this.inputDomNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	if(isChecked) {
 		this.inputDomNode.checked = true;
 	}

core/modules/widgets/range.js

@@ -50,6 +50,10 @@ RangeWidget.prototype.render = function(parent,nextSibling) {
 		this.inputDomNode.setAttribute("disabled",true);
 	}
 	this.inputDomNode.value = this.getValue();
+	this.assignAttributes(this.inputDomNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
 	// Add a click event handler
 	$tw.utils.addEventListeners(this.inputDomNode,[
 		{name:"mousedown", handlerObject:this, handlerMethod:"handleMouseDownEvent"},

core/modules/widgets/scrollable.js

@@ -12,6 +12,8 @@ Scrollable widget
 /*global $tw: false */
 "use strict";
 
+var DEBOUNCE_INTERVAL = 100; // Delay after last scroll event before updating the bound tiddler
+
 var Widget = require("$:/core/modules/widgets/widget.js").widget;
 
 var ScrollableWidget = function(parseTreeNode,options) {
@@ -171,6 +173,53 @@ ScrollableWidget.prototype.render = function(parent,nextSibling) {
 	parent.insertBefore(this.outerDomNode,nextSibling);
 	this.renderChildren(this.innerDomNode,null);
 	this.domNodes.push(this.outerDomNode);
+	// If the scroll position is bound to a tiddler
+	if(this.scrollableBind) {
+		// After a delay for rendering, scroll to the bound position
+		this.updateScrollPositionFromBoundTiddler();
+		// Set up event listener
+		this.currentListener = this.listenerFunction.bind(this);
+		this.outerDomNode.addEventListener("scroll", this.currentListener);
+	}
+};
+
+ScrollableWidget.prototype.listenerFunction = function(event) {
+	self = this;
+	clearTimeout(this.timeout);
+	this.timeout = setTimeout(function() {
+		var existingTiddler = self.wiki.getTiddler(self.scrollableBind),
+			newTiddlerFields = {
+				title: self.scrollableBind,
+				"scroll-left": self.outerDomNode.scrollLeft.toString(),
+				"scroll-top": self.outerDomNode.scrollTop.toString()
+			};
+		if(!existingTiddler || (existingTiddler.fields["title"] !== newTiddlerFields["title"]) || (existingTiddler.fields["scroll-left"] !== newTiddlerFields["scroll-left"] || existingTiddler.fields["scroll-top"] !== newTiddlerFields["scroll-top"])) {
+			self.wiki.addTiddler(new $tw.Tiddler(existingTiddler,newTiddlerFields));
+		}
+	}, DEBOUNCE_INTERVAL);
+}
+
+ScrollableWidget.prototype.updateScrollPositionFromBoundTiddler = function() {
+	// Bail if we're running on the fakedom
+	if(!this.outerDomNode.scrollTo) {
+		return;
+	}
+	var tiddler = this.wiki.getTiddler(this.scrollableBind);
+	if(tiddler) {
+		var scrollLeftTo = this.outerDomNode.scrollLeft;
+		if(parseFloat(tiddler.fields["scroll-left"]).toString() === tiddler.fields["scroll-left"]) {
+			scrollLeftTo = parseFloat(tiddler.fields["scroll-left"]);
+		}
+		var scrollTopTo = this.outerDomNode.scrollTop;
+		if(parseFloat(tiddler.fields["scroll-top"]).toString() === tiddler.fields["scroll-top"]) {
+			scrollTopTo = parseFloat(tiddler.fields["scroll-top"]);
+		}
+		this.outerDomNode.scrollTo({
+			top: scrollTopTo,
+			left: scrollLeftTo,
+			behavior: "instant"
+		})
+	}
 };
 
 /*
@@ -178,6 +227,7 @@ Compute the internal state of the widget
 */
 ScrollableWidget.prototype.execute = function() {
 	// Get attributes
+	this.scrollableBind = this.getAttribute("bind");
 	this.fallthrough = this.getAttribute("fallthrough","yes");
 	this["class"] = this.getAttribute("class");
 	// Make child widgets
@@ -193,7 +243,22 @@ ScrollableWidget.prototype.refresh = function(changedTiddlers) {
 		this.refreshSelf();
 		return true;
 	}
-	return this.refreshChildren(changedTiddlers);
+	// If the bound tiddler has changed, update the eventListener and update scroll position
+	if(changedAttributes["bind"]) {
+		if(this.currentListener) {
+			this.outerDomNode.removeEventListener("scroll", this.currentListener, false);
+		}
+		this.scrollableBind = this.getAttribute("bind");
+		this.currentListener = this.listenerFunction.bind(this);
+		this.outerDomNode.addEventListener("scroll", this.currentListener);
+	}
+	// Refresh children
+	var result = this.refreshChildren(changedTiddlers);
+	// If the bound tiddler has changed, update scroll position
+	if(changedAttributes["bind"] || changedTiddlers[this.getAttribute("bind")]) {
+		this.updateScrollPositionFromBoundTiddler();
+	}
+	return result;
 };
 
 exports.scrollable = ScrollableWidget;

core/modules/widgets/select.js

@@ -40,7 +40,31 @@ SelectWidget.prototype.render = function(parent,nextSibling) {
 	this.parentDomNode = parent;
 	this.computeAttributes();
 	this.execute();
-	this.renderChildren(parent,nextSibling);
+	//Create element
+	var domNode = this.document.createElement("select");
+	if(this.selectClass) {
+		domNode.className = this.selectClass;
+	}
+	// Assign data- attributes
+	this.assignAttributes(domNode,{
+		sourcePrefix: "data-",
+		destPrefix: "data-"
+	});
+	if(this.selectMultiple) {
+		domNode.setAttribute("multiple","multiple");
+	}
+	if(this.selectSize) {
+		domNode.setAttribute("size",this.selectSize);
+	}
+	if(this.selectTabindex) {
+		domNode.setAttribute("tabindex",this.selectTabindex);
+	}
+	if(this.selectTooltip) {
+		domNode.setAttribute("title",this.selectTooltip);
+	}
+	this.parentDomNode.insertBefore(domNode,nextSibling);
+	this.renderChildren(domNode,null);
+	this.domNodes.push(domNode);
 	this.setSelectValue();
 	if(this.selectFocus == "yes") {
 		this.getSelectDomNode().focus();
@@ -113,7 +137,7 @@ SelectWidget.prototype.setSelectValue = function() {
 Get the DOM node of the select element
 */
 SelectWidget.prototype.getSelectDomNode = function() {
-	return this.children[0].domNodes[0];
+	return this.domNodes[0];
 };
 
 // Return an array of the selected opion values
@@ -149,27 +173,7 @@ SelectWidget.prototype.execute = function() {
 	this.selectTooltip = this.getAttribute("tooltip");
 	this.selectFocus = this.getAttribute("focus");
 	// Make the child widgets
-	var selectNode = {
+	this.makeChildWidgets();
-		type: "element",
-		tag: "select",
-		children: this.parseTreeNode.children
-	};
-	if(this.selectClass) {
-		$tw.utils.addAttributeToParseTreeNode(selectNode,"class",this.selectClass);
-	}
-	if(this.selectMultiple) {
-		$tw.utils.addAttributeToParseTreeNode(selectNode,"multiple","multiple");
-	}
-	if(this.selectSize) {
-		$tw.utils.addAttributeToParseTreeNode(selectNode,"size",this.selectSize);
-	}
-	if(this.selectTabindex) {
-		$tw.utils.addAttributeToParseTreeNode(selectNode,"tabindex",this.selectTabindex);
-	}
-	if(this.selectTooltip) {
-		$tw.utils.addAttributeToParseTreeNode(selectNode,"title",this.selectTooltip);
-	}
-	this.makeChildWidgets([selectNode]);
 };
 
 /*
@@ -178,17 +182,21 @@ Selectively refreshes the widget if needed. Returns true if the widget or any of
 SelectWidget.prototype.refresh = function(changedTiddlers) {
 	var changedAttributes = this.computeAttributes();
 	// If we're using a different tiddler/field/index then completely refresh ourselves
-	if(changedAttributes.tiddler || changedAttributes.field || changedAttributes.index || changedAttributes.tooltip) {
+	if(changedAttributes.tiddler || changedAttributes.field || changedAttributes.index || changedAttributes.tooltip || changedAttributes.tabindex) {
 		this.refreshSelf();
 		return true;
-	// If the target tiddler value has changed, just update setting and refresh the children
 	} else {
 		if(changedAttributes.class) {
 			this.selectClass = this.getAttribute("class");
 			this.getSelectDomNode().setAttribute("class",this.selectClass); 
 		}
-		
+		this.assignAttributes(this.getSelectDomNode(),{
+			changedAttributes: changedAttributes,
+			sourcePrefix: "data-",
+			destPrefix: "data-"
+		});
 		var childrenRefreshed = this.refreshChildren(changedTiddlers);
+		// If the target tiddler value has changed, just update setting and refresh the children
 		if(changedTiddlers[this.selectTitle] || childrenRefreshed) {
 			this.setSelectValue();
 		} 

core/modules/widgets/testcase.js

@@ -0,0 +1,165 @@
+/*\
+title: $:/core/modules/widgets/testcase.js
+type: application/javascript
+module-type: widget
+
+Widget to display a test case
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+var Widget = require("$:/core/modules/widgets/widget.js").widget;
+
+var TestCaseWidget = function(parseTreeNode,options) {
+	this.initialise(parseTreeNode,options);
+};
+
+/*
+Inherit from the base widget class
+*/
+TestCaseWidget.prototype = new Widget();
+
+/*
+Render this widget into the DOM
+*/
+TestCaseWidget.prototype.render = function(parent,nextSibling) {
+	var self = this;
+	this.parentDomNode = parent;
+	this.computeAttributes();
+	this.execute();
+	// Create container DOM node
+	var domNode = this.document.createElement("div");
+	this.domNodes.push(domNode);
+	parent.insertBefore(domNode,nextSibling);
+	// Render the children into a hidden DOM node
+	var parser = {
+		tree: [{
+			type: "widget",
+			attributes: {},
+			orderedAttributes: [],
+			children: this.parseTreeNode.children || []
+		}]
+	};
+	this.contentRoot = this.wiki.makeWidget(parser,{
+		document: $tw.fakeDocument,
+		parentWidget: this
+	});
+	this.contentContainer = $tw.fakeDocument.createElement("div");
+	this.contentRoot.render(this.contentContainer,null);
+	// Create a wiki
+	this.testcaseWiki = new $tw.Wiki();
+	// Always load the core plugin
+	var loadTiddler = function(title) {
+		var tiddler = self.wiki.getTiddler(title);
+		if(tiddler) {
+			self.testcaseWiki.addTiddler(tiddler);
+		}
+	}
+	loadTiddler("$:/core");
+	loadTiddler("$:/plugins/tiddlywiki/codemirror");
+	// Load tiddlers from child data widgets
+	var tiddlers = [];
+	this.findChildrenDataWidgets(this.contentRoot.children,"data",function(widget) {
+		Array.prototype.push.apply(tiddlers,widget.readDataTiddlerValues());
+	});
+	var jsonPayload = JSON.stringify(tiddlers);
+	this.testcaseWiki.addTiddlers(tiddlers);
+	// Unpack plugin tiddlers
+	this.testcaseWiki.readPluginInfo();
+	this.testcaseWiki.registerPluginTiddlers("plugin");
+	this.testcaseWiki.unpackPluginTiddlers();
+	this.testcaseWiki.addIndexersToWiki();
+	// Generate a `transclusion` variable that depends on the values of the payload tiddlers so that the template can easily make unique state tiddlers
+	this.setVariable("transclusion",$tw.utils.hashString(jsonPayload));
+	// Generate a `payloadTiddlers` variable that contains the payload in JSON format
+	this.setVariable("payloadTiddlers",jsonPayload);
+	// Only run the tests if the testcase output and expected results were specified, and those tiddlers actually exist in the wiki
+	var shouldRunTests = false;
+	if(this.testcaseTestOutput && this.testcaseWiki.tiddlerExists(this.testcaseTestOutput) && this.testcaseTestExpectedResult && this.testcaseWiki.tiddlerExists(this.testcaseTestExpectedResult)) {
+		shouldRunTests = true;
+	}
+	// Render the test rendering if required
+	if(shouldRunTests) {
+		var testcaseOutputContainer = $tw.fakeDocument.createElement("div");
+		var testcaseOutputWidget = this.testcaseWiki.makeTranscludeWidget(this.testcaseTestOutput,{
+			document: $tw.fakeDocument,
+			parseAsInline: false,
+			parentWidget: this,
+			variables: {
+				currentTiddler: this.testcaseTestOutput
+			}
+		});
+		testcaseOutputWidget.render(testcaseOutputContainer);
+	}
+	// Clear changes queue
+	this.testcaseWiki.clearTiddlerEventQueue();
+	// Run the actions if provided
+	if(this.testcaseWiki.tiddlerExists(this.testcaseTestActions)) {
+		testcaseOutputWidget.invokeActionString(this.testcaseWiki.getTiddlerText(this.testcaseTestActions));
+		testcaseOutputWidget.refresh(this.testcaseWiki.changedTiddlers,testcaseOutputContainer);
+	}
+	// Set up the test result variables
+	var testResult = "",
+		outputHTML = "",
+		expectedHTML = "";
+	if(shouldRunTests) {
+		outputHTML = testcaseOutputContainer.children[0].innerHTML;
+		expectedHTML = this.testcaseWiki.getTiddlerText(this.testcaseTestExpectedResult);
+		if(outputHTML === expectedHTML) {
+			testResult = "pass";
+		} else {
+			testResult = "fail";
+		}
+		this.setVariable("outputHTML",outputHTML);
+		this.setVariable("expectedHTML",expectedHTML);
+		this.setVariable("testResult",testResult);
+		this.setVariable("currentTiddler",this.testcaseTestOutput);
+	}
+	// Don't display anything if testHideIfPass is "yes" and the tests have passed
+	if(this.testcaseHideIfPass === "yes" && testResult !== "fail") {
+		return;
+	}
+	// Render the page root template of the subwiki
+	var rootWidget = this.testcaseWiki.makeTranscludeWidget(this.testcaseTemplate,{
+		document: this.document,
+		parseAsInline: false,
+		parentWidget: this
+	});
+	rootWidget.render(domNode);
+	// Trap changes in the wiki and refresh the rendering
+	this.testcaseWiki.addEventListener("change",function(changes) {
+		rootWidget.refresh(changes,domNode);
+	});
+};
+
+/*
+Compute the internal state of the widget
+*/
+TestCaseWidget.prototype.execute = function() {
+	this.testcaseTemplate = this.getAttribute("template","$:/core/ui/testcases/DefaultTemplate");
+	this.testcaseTestOutput = this.getAttribute("testOutput");
+	this.testcaseTestActions = this.getAttribute("testActions");
+	this.testcaseTestExpectedResult = this.getAttribute("testExpectedResult");
+	this.testcaseHideIfPass = this.getAttribute("testHideIfPass");
+};
+
+/*
+Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
+*/
+TestCaseWidget.prototype.refresh = function(changedTiddlers) {
+	var changedAttributes = this.computeAttributes();
+	if($tw.utils.count(changedAttributes) > 0) {
+		this.refreshSelf();
+		return true;
+	} else {
+		return this.contentRoot.refresh(changedTiddlers);
+	}
+};
+
+exports["testcase"] = TestCaseWidget;
+
+})();

core/modules/widgets/transclude.js

@@ -30,7 +30,30 @@ TranscludeWidget.prototype.render = function(parent,nextSibling) {
 	this.parentDomNode = parent;
 	this.computeAttributes();
 	this.execute();
+	try {
 		this.renderChildren(parent,nextSibling);
+	} catch(error) {
+		if(error instanceof $tw.utils.TranscludeRecursionError) {
+			// We were infinite looping.
+			// We need to try and abort as much of the loop as we can, so we will keep "throwing" upward until we find a transclusion that has a different signature.
+			// Hopefully that will land us just outside where the loop began. That's where we want to issue an error.
+			// Rendering widgets beneath this point may result in a freezing browser if they explode exponentially.
+			var transcludeSignature = this.getVariable("transclusion");
+			if(this.getAncestorCount() > $tw.utils.TranscludeRecursionError.MAX_WIDGET_TREE_DEPTH - 50) {
+				// For the first fifty transcludes we climb up, we simply collect signatures.
+				// We're assuming that those first 50 will likely include all transcludes involved in the loop.
+				error.signatures[transcludeSignature] = true;
+			} else if(!error.signatures[transcludeSignature]) {
+				// Now that we're past the first 50, let's look for the first signature that wasn't in the loop. That'll be where we print the error and resume rendering.
+				this.children = [this.makeChildWidget({type: "error", attributes: {
+					"$message": {type: "string", value: $tw.language.getString("Error/RecursiveTransclusion")}
+				}})];
+				this.renderChildren(parent,nextSibling);
+				return;
+			}
+		}
+		throw error;
+	}
 };
 
 /*
@@ -109,6 +132,7 @@ TranscludeWidget.prototype.collectAttributes = function() {
 		this.recursionMarker = this.getAttribute("recursionMarker","yes");
 	} else {
 		this.transcludeVariable = this.getAttribute("$variable");
+		this.transcludeVariableIsFunction = false;
 		this.transcludeType = this.getAttribute("$type");
 		this.transcludeOutput = this.getAttribute("$output","text/html");
 		this.transcludeTitle = this.getAttribute("$tiddler",this.getVariable("currentTiddler"));
@@ -184,7 +208,9 @@ TranscludeWidget.prototype.getTransclusionTarget = function() {
 		if(this.transcludeVariable) {
 			// Transcluding a variable
 			var variableInfo = this.getVariableInfo(this.transcludeVariable,{params: this.getOrderedTransclusionParameters()});
+			this.transcludeVariableIsFunction = variableInfo.srcVariable && variableInfo.srcVariable.isFunctionDefinition;
 			text = variableInfo.text;
+			this.transcludeFunctionResult = text;
 			return {
 				text: variableInfo.text,
 				type: this.transcludeType
@@ -219,21 +245,24 @@ TranscludeWidget.prototype.parseTransclusionTarget = function(parseAsInline) {
 			// Transcluding a variable
 			var variableInfo = this.getVariableInfo(this.transcludeVariable,{params: this.getOrderedTransclusionParameters()}),
 				srcVariable = variableInfo && variableInfo.srcVariable;
+			if(srcVariable && srcVariable.isFunctionDefinition) {
+				this.transcludeVariableIsFunction = true;
+				this.transcludeFunctionResult = (variableInfo.resultList ? variableInfo.resultList[0] : variableInfo.text) || "";
+			}
 			if(variableInfo.text) {
 				if(srcVariable && srcVariable.isFunctionDefinition) {
-					var result = (variableInfo.resultList ? variableInfo.resultList[0] : variableInfo.text) || "";
 					parser = {
 						tree: [{
 							type: "text",
-							text: result
+							text: this.transcludeFunctionResult
 						}],
-						source: result,
+						source: this.transcludeFunctionResult,
 						type: "text/vnd.tiddlywiki"
 					};
 					if(parseAsInline) {
 						parser.tree[0] = {
 							type: "text",
-							text: result
+							text: this.transcludeFunctionResult
 						};
 					} else {
 						parser.tree[0] = {
@@ -241,7 +270,7 @@ TranscludeWidget.prototype.parseTransclusionTarget = function(parseAsInline) {
 							tag: "p",
 							children: [{
 								type: "text",
-								text: result
+								text: this.transcludeFunctionResult
 							}]
 						}
 					}
@@ -430,12 +459,19 @@ TranscludeWidget.prototype.parserNeedsRefresh = function() {
 	return (this.sourceText === undefined || parserInfo.sourceText !== this.sourceText || parserInfo.parserType !== this.parserType)
 };
 
+TranscludeWidget.prototype.functionNeedsRefresh = function() {
+	var oldResult = this.transcludeFunctionResult;
+	var variableInfo = this.getVariableInfo(this.transcludeVariable,{params: this.getOrderedTransclusionParameters()});
+	var newResult = (variableInfo.resultList ? variableInfo.resultList[0] : variableInfo.text) || "";
+	return oldResult !== newResult;
+}
+
 /*
 Selectively refreshes the widget if needed. Returns true if the widget or any of its children needed re-rendering
 */
 TranscludeWidget.prototype.refresh = function(changedTiddlers) {
 	var changedAttributes = this.computeAttributes();
-	if(($tw.utils.count(changedAttributes) > 0) || (!this.transcludeVariable && changedTiddlers[this.transcludeTitle] && this.parserNeedsRefresh())) {
+	if(($tw.utils.count(changedAttributes) > 0) || (this.transcludeVariableIsFunction && this.functionNeedsRefresh()) || (!this.transcludeVariable && changedTiddlers[this.transcludeTitle] && this.parserNeedsRefresh())) {
 		this.refreshSelf();
 		return true;
 	} else {

core/modules/widgets/widget.js

@@ -12,9 +12,6 @@ Widget base class
 /*global $tw: false */
 "use strict";
 
-/* Maximum permitted depth of the widget tree for recursion detection */
-var MAX_WIDGET_TREE_DEPTH = 1000;
-
 /*
 Create a widget object for a parse tree node
 	parseTreeNode: reference to the parse tree node to be rendered
@@ -153,7 +150,7 @@ Widget.prototype.getVariableInfo = function(name,options) {
 		} else if(variable.isFunctionDefinition) {
 			// Function evaluations
 			params = self.resolveVariableParameters(variable.params,actualParams);
-			var variables = Object.create(null);
+			var variables = options.variables || Object.create(null);
 			// Apply default parameter values
 			$tw.utils.each(variable.params,function(param,index) {
 				if(param["default"]) {
@@ -166,6 +163,8 @@ Widget.prototype.getVariableInfo = function(name,options) {
 			});
 			resultList = this.wiki.filterTiddlers(value,this.makeFakeWidgetWithVariables(variables),options.source);
 			value = resultList[0] || "";
+		} else {
+			params = variable.params;
 		}
 		return {
 			text: value,
@@ -317,7 +316,8 @@ Widget.prototype.getStateQualifier = function(name) {
 Make a fake widget with specified variables, suitable for variable lookup in filters
 */
 Widget.prototype.makeFakeWidgetWithVariables = function(variables) {
-	var self = this;
+	var self = this,
+		variables = variables || {};
 	return {
 		getVariable: function(name,opts) {
 			if($tw.utils.hop(variables,name)) {
@@ -335,7 +335,7 @@ Widget.prototype.makeFakeWidgetWithVariables = function(variables) {
 				};
 			} else {
 				opts = opts || {};
-				opts.variables = variables;
+				opts.variables = $tw.utils.extend(variables,opts.variables);
 				return self.getVariableInfo(name,opts);
 			};
 		},
@@ -413,16 +413,34 @@ Widget.prototype.getAttribute = function(name,defaultText) {
 };
 
 /*
-Assign the computed attributes of the widget to a domNode
+Assign the common attributes of the widget to a domNode
 options include:
-excludeEventAttributes: ignores attributes whose name begins with "on"
+sourcePrefix: prefix of attributes that are to be directly assigned (defaults to the empty string meaning all attributes)
+destPrefix: prefix to be applied to attribute names that are to be directly assigned (defaults to the emtpy string which means no prefix is added)
+changedAttributes: hashmap by attribute name of attributes to process (if missing, process all attributes)
+excludeEventAttributes: ignores attributes whose name would begin with "on"
 */
 Widget.prototype.assignAttributes = function(domNode,options) {
 	options = options || {};
-	var self = this;
+	var self = this,
+		changedAttributes = options.changedAttributes || this.attributes,
+		sourcePrefix = options.sourcePrefix || "",
+		destPrefix = options.destPrefix || "",
+		EVENT_ATTRIBUTE_PREFIX = "on";
 	var assignAttribute = function(name,value) {
+		// Process any style attributes before considering sourcePrefix and destPrefix
+		if(name.substr(0,6) === "style." && name.length > 6) {
+			domNode.style[$tw.utils.unHyphenateCss(name.substr(6))] = value;
+			return;
+		}
+		// Check if the sourcePrefix is a match
+		if(name.substr(0,sourcePrefix.length) === sourcePrefix) {
+			name = destPrefix + name.substr(sourcePrefix.length);
+		} else {
+			value = undefined;
+		}
 		// Check for excluded attribute names
-		if(options.excludeEventAttributes && name.substr(0,2) === "on") {
+		if(options.excludeEventAttributes && name.substr(0,2).toLowerCase() === EVENT_ATTRIBUTE_PREFIX) {
 			value = undefined;
 		}
 		if(value !== undefined) {
@@ -432,26 +450,24 @@ Widget.prototype.assignAttributes = function(domNode,options) {
 				namespace = "http://www.w3.org/1999/xlink";
 				name = name.substr(6);
 			}
-			// Handle styles
-			if(name.substr(0,6) === "style." && name.length > 6) {
-				domNode.style[$tw.utils.unHyphenateCss(name.substr(6))] = value;
-			} else {
 			// Setting certain attributes can cause a DOM error (eg xmlns on the svg element)
 			try {
 				domNode.setAttributeNS(namespace,name,value);
 			} catch(e) {
 			}
 		}
-		}
+	};
-	}
+	// If the parse tree node has the orderedAttributes property then use that order
-	// Not all parse tree nodes have the orderedAttributes property
 	if(this.parseTreeNode.orderedAttributes) {
 		$tw.utils.each(this.parseTreeNode.orderedAttributes,function(attribute,index) {
-			assignAttribute(attribute.name,self.attributes[attribute.name]);
+			if(attribute.name in changedAttributes) {
+				assignAttribute(attribute.name,self.getAttribute(attribute.name));
+			}
 		});
+	// Otherwise update each changed attribute irrespective of order
 	} else {
-		$tw.utils.each(Object.keys(self.attributes).sort(),function(name) {
+		$tw.utils.each(changedAttributes,function(value,name) {
-			assignAttribute(name,self.attributes[name]);
+			assignAttribute(name,self.getAttribute(name));
 		});	
 	}
 };
@@ -478,10 +494,8 @@ Widget.prototype.makeChildWidgets = function(parseTreeNodes,options) {
 	this.children = [];
 	var self = this;
 	// Check for too much recursion
-	if(this.getAncestorCount() > MAX_WIDGET_TREE_DEPTH) {
+	if(this.getAncestorCount() > $tw.utils.TranscludeRecursionError.MAX_WIDGET_TREE_DEPTH) {
-		this.children.push(this.makeChildWidget({type: "error", attributes: {
+		throw new $tw.utils.TranscludeRecursionError();
-			"$message": {type: "string", value: $tw.language.getString("Error/RecursiveTransclusion")}
-		}}));
 	} else {
 		// Create set variable widgets for each variable
 		$tw.utils.each(options.variables,function(value,name) {
@@ -797,6 +811,21 @@ Widget.prototype.allowActionPropagation = function() {
 	return true;
 };
 
+/*
+Find child <$data> widgets recursively. The tag name allows aliased versions of the widget to be found too
+*/
+Widget.prototype.findChildrenDataWidgets = function(children,tag,callback) {
+	var self = this;
+	$tw.utils.each(children,function(child) {
+		if(child.dataWidgetTag === tag) {
+			callback(child);
+		}
+		if(child.children) {
+			self.findChildrenDataWidgets(child.children,tag,callback);
+		}
+	});
+};
+
 /*
 Evaluate a variable with parameters. This is a static convenience method that attempts to evaluate a variable as a function, returning an array of strings
 */

core/modules/wiki.js

@@ -534,8 +534,8 @@ Return an array of tiddler titles that link to the specified tiddler
 */
 exports.getTiddlerBacklinks = function(targetTitle) {
 	var self = this,
-		backlinksIndexer = this.getIndexer("BacklinksIndexer"),
+		backIndexer = this.getIndexer("BackIndexer"),
-		backlinks = backlinksIndexer && backlinksIndexer.lookup(targetTitle);
+		backlinks = backIndexer && backIndexer.subIndexers.link.lookup(targetTitle);
 
 	if(!backlinks) {
 		backlinks = [];
@@ -549,6 +549,68 @@ exports.getTiddlerBacklinks = function(targetTitle) {
 	return backlinks;
 };
 
+
+/*
+Return an array of tiddler titles that are directly transcluded within the given parse tree
+ */
+exports.extractTranscludes = function(parseTreeRoot) {
+	// Count up the transcludes
+	var transcludes = [],
+		checkParseTree = function(parseTree, parentNode) {
+			for(var t=0; t<parseTree.length; t++) {
+				var parseTreeNode = parseTree[t];
+				if(parseTreeNode.type === "transclude" && parseTreeNode.attributes.$tiddler && parseTreeNode.attributes.$tiddler.type === "string") {
+					var value;
+					// if it is Transclusion with Templates like `{{Index||$:/core/ui/TagTemplate}}`, the `$tiddler` will point to the template. We need to find the actual target tiddler from parent node
+					if(parentNode && parentNode.type === "tiddler" && parentNode.attributes.tiddler && parentNode.attributes.tiddler.type === "string") {
+						value = parentNode.attributes.tiddler.value;
+					} else {
+						value = parseTreeNode.attributes.$tiddler.value;
+					}
+					if(transcludes.indexOf(value) === -1) {
+						transcludes.push(value);
+					}
+				}
+				if(parseTreeNode.children) {
+					checkParseTree(parseTreeNode.children, parseTreeNode);
+				}
+			}
+		};
+	checkParseTree(parseTreeRoot);
+	return transcludes;
+};
+
+
+/*
+Return an array of tiddler titles that are transcluded from the specified tiddler
+*/
+exports.getTiddlerTranscludes = function(title) {
+	var self = this;
+	// We'll cache the transcludes so they only get computed if the tiddler changes
+	return this.getCacheForTiddler(title,"transcludes",function() {
+		// Parse the tiddler
+		var parser = self.parseTiddler(title);
+		if(parser) {
+			return self.extractTranscludes(parser.tree);
+		}
+		return [];
+	});
+};
+
+/*
+Return an array of tiddler titles that transclude to the specified tiddler
+*/
+exports.getTiddlerBacktranscludes = function(targetTitle) {
+	var self = this,
+		backIndexer = this.getIndexer("BackIndexer"),
+		backtranscludes = backIndexer && backIndexer.subIndexers.transclude.lookup(targetTitle);
+
+	if(!backtranscludes) {
+		backtranscludes = [];
+	}
+	return backtranscludes;
+};
+
 /*
 Return a hashmap of tiddler titles that are referenced but not defined. Each value is the number of times the missing tiddler is referenced
 */
@@ -1287,7 +1349,7 @@ exports.search = function(text,options) {
 			console.log("Regexp error parsing /(" + text + ")/" + flags + ": ",e);
 		}
 	} else if(options.some) {
-		terms = text.trim().split(/ +/);
+		terms = text.trim().split(/[^\S\xA0]+/);
 		if(terms.length === 1 && terms[0] === "") {
 			searchTermsRegExps = null;
 		} else {
@@ -1298,7 +1360,7 @@ exports.search = function(text,options) {
 			searchTermsRegExps.push(new RegExp("(" + regExpStr + ")",flags));
 		}
 	} else { // default: words
-		terms = text.split(/ +/);
+		terms = text.split(/[^\S\xA0]+/);
 		if(terms.length === 1 && terms[0] === "") {
 			searchTermsRegExps = null;
 		} else {

core/palettes/Vanilla.tid

@@ -95,6 +95,9 @@ table-footer-background: #a8a8a8
 table-header-background: #f0f0f0
 tag-background: #ec6
 tag-foreground: #ffffff
+testcase-accent-level-1: #84C5E6
+testcase-accent-level-2: #E3B740
+testcase-accent-level-3: #5FD564
 tiddler-background: <<colour background>>
 tiddler-border: <<colour background>>
 tiddler-controls-foreground-hover: #888888

core/plugin.info

@@ -5,5 +5,6 @@
 	"author": "JeremyRuston",
 	"core-version": ">=5.0.0",
 	"plugin-priority": "0",
-	"list": "readme"
+	"list": "readme",
+	"stability": "STABILITY_2_STABLE"
 }

core/templates/external-js/save-all-external-js.tid

@@ -3,7 +3,7 @@ title: $:/core/save/all-external-js
 \whitespace trim
 \import [subfilter{$:/core/config/GlobalImportFilter}]
 \define saveTiddlerFilter()
-[is[tiddler]] -[prefix[$:/state/popup/]] -[prefix[$:/temp/]] -[prefix[$:/HistoryList]] -[status[pending]plugin-type[import]] -[[$:/core]] -[[$:/boot/boot.css]] -[type[application/javascript]library[yes]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] +[sort[title]] $(publishFilter)$
+[is[tiddler]] -[prefix[$:/state/popup/]] -[prefix[$:/temp/]] -[prefix[$:/HistoryList]] -[status[pending]plugin-type[import]] -[[$:/core]] -[[$:/boot/boot.css]] -[is[system]type[application/javascript]library[yes]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] +[sort[title]] $(publishFilter)$
 \end
 
 <!-- Important: core library is provided by serving URI encoded $:/core/templates/tiddlywiki5.js -->

core/templates/external-js/save-offline-external-js.tid

@@ -3,7 +3,7 @@ title: $:/core/save/offline-external-js
 \whitespace trim
 \import [subfilter{$:/core/config/GlobalImportFilter}]
 \define saveTiddlerFilter()
-[is[tiddler]] -[prefix[$:/state/popup/]] -[prefix[$:/temp/]] -[prefix[$:/HistoryList]] -[status[pending]plugin-type[import]] -[[$:/core]] -[[$:/plugins/tiddlywiki/filesystem]] -[[$:/plugins/tiddlywiki/tiddlyweb]] -[[$:/boot/boot.css]] -[type[application/javascript]library[yes]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] +[sort[title]] $(publishFilter)$
+[is[tiddler]] -[prefix[$:/state/popup/]] -[prefix[$:/temp/]] -[prefix[$:/HistoryList]] -[status[pending]plugin-type[import]] -[[$:/core]] -[[$:/plugins/tiddlywiki/filesystem]] -[[$:/plugins/tiddlywiki/tiddlyweb]] -[[$:/boot/boot.css]] -[is[system]type[application/javascript]library[yes]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] +[sort[title]] $(publishFilter)$
 \end
 \define defaultCoreURL() tiddlywikicore-$(version)$.js
 <$let coreURL={{{ [[coreURL]is[variable]then<coreURL>else<defaultCoreURL>] }}}>