Add indexes to the wiki store to improve performance (#3951)

* First pass at modular wiki indexes

An exploratory experiment

* Fix tests

* Faster checking for existence of index methods

We don't really need to check the type

* Use the index for the has operator

* Fix typo

* Move iterator index methods into indexer modules

Now boot.js doesn't know the core indexers

* Fix up the other iterator index functions

* Fix crash with missing index branch

* Limit the field indexer to values less than 128 characters

* Fallback to the old manual scan if the index method returns null
* Sadly, we can no longe re-use the field indexer to accelerate the `has` operator, because the index now omits tiddlers that have field values longer than the limit

Still need to make the index configuration exposed somehow

* Rearrange tests so that we can test with and without indexers

We also need to expose the list of enabled indexers as a config option

* Test the field indexer with different length fields

So that we test the indexed and non-indexed codepaths

boot/boot.js

@@ -1035,6 +1035,22 @@ $tw.Wiki = function(options) {
 				shadowTiddlerTitles = Object.keys(shadowTiddlers);
 			}
 			return shadowTiddlerTitles;
+		},
+		enableIndexers = options.enableIndexers || null, // Array of indexer names to enable, or null to use all available indexers
+		indexers = [],
+		indexersByName = Object.create(null);
+
+	this.addIndexer = function(indexer,name) {
+		// Bail if this indexer is not enabled
+		if(enableIndexers && enableIndexers.indexOf(name) === -1) {
+			return;
+		}
+		indexers.push(indexer);
+		indexersByName[name] = indexer;
+	};
+
+	this.getIndexer = function(name) {
+		return indexersByName[name] || null;
 	};
 
 	// Add a tiddler to the store
@@ -1046,6 +1062,7 @@ $tw.Wiki = function(options) {
 		if(tiddler) {
 			var title = tiddler.fields.title;
 			if(title) {
+				var oldTiddler = this.getTiddler(title);
 // Uncomment the following line for detailed logs of all tiddler writes
 // console.log("Adding",title,tiddler)
 				tiddlers[title] = tiddler;
@@ -1054,6 +1071,9 @@ $tw.Wiki = function(options) {
 				}
 				this.clearCache(title);
 				this.clearGlobalCache();
+				$tw.utils.each(indexers,function(indexer) {
+					indexer.update(oldTiddler,tiddler);
+				});
 				this.enqueueTiddlerEvent(title);
 			}
 		}
@@ -1064,6 +1084,7 @@ $tw.Wiki = function(options) {
 // Uncomment the following line for detailed logs of all tiddler deletions
 // console.log("Deleting",title)
 		if($tw.utils.hop(tiddlers,title)) {
+			var oldTiddler = this.getTiddler(title);
 			delete tiddlers[title];
 			if(tiddlerTitles) {
 				var index = tiddlerTitles.indexOf(title);
@@ -1073,6 +1094,10 @@ $tw.Wiki = function(options) {
 			}
 			this.clearCache(title);
 			this.clearGlobalCache();
+			var newTiddler = this.getTiddler(title);
+			$tw.utils.each(indexers,function(indexer) {
+				indexer.update(oldTiddler,newTiddler);
+			});
 			this.enqueueTiddlerEvent(title,true);
 		}
 	};
@@ -1159,7 +1184,6 @@ $tw.Wiki = function(options) {
 				callback(tiddlers[title],title);
 			}
 		}
-
 	};
 
 	// Test for the existence of a tiddler (excludes shadow tiddlers)
@@ -1273,8 +1297,14 @@ $tw.Wiki = function(options) {
 		shadowTiddlerTitles = null;
 		this.clearCache(null);
 		this.clearGlobalCache();
+		$tw.utils.each(indexers,function(indexer) {
+			indexer.rebuild();
+		});
 	};
 
+	if(this.addIndexersToWiki) {
+		this.addIndexersToWiki();
+	}
 };
 
 // Dummy methods that will be filled in after boot

core/modules/filters/field.js

@@ -16,7 +16,7 @@ Filter operator for comparing fields for equality
 Export our filter function
 */
 exports.field = function(source,operator,options) {
-	var results = [],
+	var results = [],indexedResults,
 		fieldname = (operator.suffix || operator.operator || "title").toLowerCase();
 	if(operator.prefix === "!") {
 		if(operator.regexp) {
@@ -53,6 +53,12 @@ exports.field = function(source,operator,options) {
 				}
 			});
 		} else {
+			if(source.byField) {
+				indexedResults = source.byField(fieldname,operator.operand);
+				if(indexedResults) {
+					return indexedResults
+				}
+			}
 			source(function(tiddler,title) {
 				if(tiddler) {
 					var text = tiddler.getFieldString(fieldname);

core/modules/filters/tag.js

@@ -16,7 +16,7 @@ Filter operator for checking for the presence of a tag
 Export our filter function
 */
 exports.tag = function(source,operator,options) {
-	var results = [];
+	var results = [],indexedResults;
 	if((operator.suffix || "").toLowerCase() === "strict" && !operator.operand) {
 		// New semantics:
 		// Always return copy of input if operator.operand is missing
@@ -25,9 +25,10 @@ exports.tag = function(source,operator,options) {
 		});
 	} else {
 		// Old semantics:
-		var tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
+		var tiddlers;
 		if(operator.prefix === "!") {
 			// Returns a copy of the input if operator.operand is missing
+			tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
 			source(function(tiddler,title) {
 				if(tiddlers.indexOf(title) === -1) {
 					results.push(title);
@@ -35,6 +36,13 @@ exports.tag = function(source,operator,options) {
 			});
 		} else {
 			// Returns empty results if operator.operand is missing
+			if(source.byTag) {
+				indexedResults = source.byTag(operator.operand);
+				if(indexedResults) {
+					return indexedResults;
+				}
+			} else {
+				tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
 				source(function(tiddler,title) {
 					if(tiddlers.indexOf(title) !== -1) {
 						results.push(title);
@@ -43,6 +51,7 @@ exports.tag = function(source,operator,options) {
 				results = options.wiki.sortByList(results,operator.operand);
 			}
 		}		
+	}
 	return results;
 };
 

core/modules/indexers/field-indexer.js

@@ -0,0 +1,141 @@
+/*\
+title: $:/core/modules/indexers/field-indexer.js
+type: application/javascript
+module-type: indexer
+
+Indexes the tiddlers with each field value
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global modules: false */
+"use strict";
+
+var DEFAULT_MAXIMUM_INDEXED_VALUE_LENGTH = 128;
+
+function FieldIndexer(wiki) {
+	this.wiki = wiki;
+	this.index = null;
+	this.maxIndexedValueLength = DEFAULT_MAXIMUM_INDEXED_VALUE_LENGTH;
+	this.addIndexMethods();
+}
+
+// Provided for testing
+FieldIndexer.prototype.setMaxIndexedValueLength = function(length) {
+	this.index = null;
+	this.maxIndexedValueLength = length;
+};
+
+FieldIndexer.prototype.addIndexMethods = function() {
+	var self = this;
+	this.wiki.each.byField = function(name,value) {
+		var titles = self.wiki.allTitles(),
+			lookup = self.lookup(name,value);
+		return lookup && lookup.filter(function(title) {
+			return titles.indexOf(title) !== -1;
+		});
+	};
+	this.wiki.eachShadow.byField = function(name,value) {
+		var titles = self.wiki.allShadowTitles(),
+			lookup = self.lookup(name,value);
+		return lookup && lookup.filter(function(title) {
+			return titles.indexOf(title) !== -1;
+		});
+	};
+	this.wiki.eachTiddlerPlusShadows.byField = function(name,value) {
+		var lookup = self.lookup(name,value);
+		return lookup ? lookup.slice(0) : null;
+	};
+	this.wiki.eachShadowPlusTiddlers.byField = function(name,value) {
+		var lookup = self.lookup(name,value);
+		return lookup ? lookup.slice(0) : null;
+	};
+};
+
+/*
+Tear down and then rebuild the index as if all tiddlers have changed
+*/
+FieldIndexer.prototype.rebuild = function() {
+	// Invalidate the index so that it will be rebuilt when it is next used
+	this.index = null;
+};
+
+/*
+Build the index for a particular field
+*/
+FieldIndexer.prototype.buildIndexForField = function(name) {
+	var self = this;
+	// Hashmap by field name of hashmap by field value of array of tiddler titles
+	this.index = this.index || Object.create(null);
+	this.index[name] = Object.create(null);
+	var baseIndex = this.index[name];
+	// Update the index for each tiddler
+	this.wiki.eachTiddlerPlusShadows(function(tiddler,title) {
+		if(name in tiddler.fields) {
+			var value = tiddler.getFieldString(name);
+			// Skip any values above the maximum length
+			if(value.length < self.maxIndexedValueLength) {
+				baseIndex[value] = baseIndex[value] || [];
+				baseIndex[value].push(title);
+			}
+		}
+	});
+};
+
+/*
+Update the index in the light of a tiddler value changing; note that the title must be identical. (Renames are handled as a separate delete and create)
+oldTiddler: old tiddler value, or null for creation
+newTiddler: new tiddler value, or null for deletion
+*/
+FieldIndexer.prototype.update = function(oldTiddler,newTiddler) {
+	var self = this;
+	// Don't do anything if the index hasn't been built yet
+	if(this.index === null) {
+		return;
+	}
+	// Remove the old tiddler from the index
+	if(oldTiddler) {
+		$tw.utils.each(this.index,function(indexEntry,name) {
+			if(name in oldTiddler.fields) {
+				var value = oldTiddler.getFieldString(name),
+					tiddlerList = indexEntry[value];
+				if(tiddlerList) {
+					var index = tiddlerList.indexOf(oldTiddler.fields.title);
+					if(index !== -1) {
+						tiddlerList.splice(index,1);
+					}
+				}
+			}
+		});
+	}
+	// Add the new tiddler to the index
+	if(newTiddler) {
+		$tw.utils.each(this.index,function(indexEntry,name) {
+			if(name in newTiddler.fields) {
+				var value = newTiddler.getFieldString(name);
+				if(value.length < self.maxIndexedValueLength) {
+					indexEntry[value] = indexEntry[value] || [];
+					indexEntry[value].push(newTiddler.fields.title);
+				}
+			}
+		});		
+	}
+};
+
+// Lookup the given field returning a list of tiddler titles
+FieldIndexer.prototype.lookup = function(name,value) {
+	// Fail the lookup if the value is too long
+	if(value.length >= this.maxIndexedValueLength) {
+		return null;
+	}
+	// Update the index if it has yet to be built
+	if(this.index === null || !this.index[name]) {
+		this.buildIndexForField(name);
+	}
+	return this.index[name][value] || [];
+};
+
+exports.FieldIndexer = FieldIndexer;
+
+})();

core/modules/indexers/tag-indexer.js

@@ -0,0 +1,131 @@
+/*\
+title: $:/core/modules/indexers/tag-indexer.js
+type: application/javascript
+module-type: indexer
+
+Indexes the tiddlers with each tag
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global modules: false */
+"use strict";
+
+function TagIndexer(wiki) {
+	this.wiki = wiki;
+	this.index = null;
+	this.addIndexMethods();
+}
+
+TagIndexer.prototype.addIndexMethods = function() {
+	var self = this;
+	this.wiki.each.byTag = function(tag) {
+		var titles = self.wiki.allTitles();
+		return self.lookup(tag).filter(function(title) {
+			return titles.indexOf(title) !== -1;
+		});
+	};
+	this.wiki.eachShadow.byTag = function(tag) {
+		var titles = self.wiki.allShadowTitles();
+		return self.lookup(tag).filter(function(title) {
+			return titles.indexOf(title) !== -1;
+		});
+	};
+	this.wiki.eachTiddlerPlusShadows.byTag = function(tag) {
+		return self.lookup(tag).slice(0);
+	};
+	this.wiki.eachShadowPlusTiddlers.byTag = function(tag) {
+		return self.lookup(tag).slice(0);
+	};
+};
+
+/*
+Tear down and then rebuild the index as if all tiddlers have changed
+*/
+TagIndexer.prototype.rebuild = function() {
+	var self = this;
+	// Hashmap by tag of array of {isSorted:, titles:[]}
+	this.index = Object.create(null);
+	// Add all the tags
+	this.wiki.eachTiddlerPlusShadows(function(tiddler,title) {
+		$tw.utils.each(tiddler.fields.tags,function(tag) {
+			if(!self.index[tag]) {
+				self.index[tag] = {isSorted: false, titles: [title]};
+			} else {
+				self.index[tag].titles.push(title);
+			}
+		});		
+	});
+};
+
+/*
+Update the index in the light of a tiddler value changing; note that the title must be identical. (Renames are handled as a separate delete and create)
+oldTiddler: old tiddler value, or null for creation
+newTiddler: new tiddler value, or null for deletion
+*/
+TagIndexer.prototype.update = function(oldTiddler,newTiddler) {
+	// Don't update the index if it has yet to be built
+	if(this.index === null) {
+		return;
+	}
+	var self = this,
+		title = oldTiddler ? oldTiddler.fields.title : newTiddler.fields.title;
+	// Handle changes to the tags
+	var oldTiddlerTags = (oldTiddler ? (oldTiddler.fields.tags || []) : []),
+		newTiddlerTags = (newTiddler ? (newTiddler.fields.tags || []) : []);
+	$tw.utils.each(oldTiddlerTags,function(oldTag) {
+		if(newTiddlerTags.indexOf(oldTag) === -1) {
+			// Deleted tag
+			var indexRecord = self.index[oldTag],
+				pos = indexRecord.titles.indexOf(title);
+			if(pos !== -1) {
+				indexRecord.titles.splice(pos,1);
+			}
+		}
+	});
+	$tw.utils.each(newTiddlerTags,function(newTag) {
+		if(oldTiddlerTags.indexOf(newTag) === -1) {
+			// New tag
+			var indexRecord = self.index[newTag];
+			if(!indexRecord) {
+				self.index[newTag] = {isSorted: false, titles: [title]};
+			} else {
+				indexRecord.titles.push(title);
+				indexRecord.isSorted = false;
+			}
+		}
+	});
+	// Handle changes to the list field of tags
+	var oldTiddlerList = (oldTiddler ? (oldTiddler.fields.list || []) : []),
+		newTiddlerList = (newTiddler ? (newTiddler.fields.list || []) : []);
+	if(!$tw.utils.isArrayEqual(oldTiddlerList,newTiddlerList)) {
+		if(self.index[title]) {
+			self.index[title].isSorted = false;			
+		}
+	}
+};
+
+// Lookup the given tag returning an ordered list of tiddler titles
+TagIndexer.prototype.lookup = function(tag) {
+	// Update the index if it has yet to be built
+	if(this.index === null) {
+		this.rebuild();
+	}
+	var indexRecord = this.index[tag];
+	if(indexRecord) {
+		if(!indexRecord.isSorted) {
+			if(this.wiki.sortByList) {
+				indexRecord.titles = this.wiki.sortByList(indexRecord.titles,tag);
+			}			
+			indexRecord.isSorted = true;
+		}
+		return indexRecord.titles;
+	} else {
+		return [];
+	}
+};
+
+exports.TagIndexer = TagIndexer;
+
+})();

core/modules/startup/load-modules.js

@@ -27,6 +27,7 @@ exports.startup = function() {
 	$tw.Tiddler.fieldModules = $tw.modules.getModulesByTypeAsHashmap("tiddlerfield");
 	$tw.modules.applyMethods("tiddlermethod",$tw.Tiddler.prototype);
 	$tw.modules.applyMethods("wikimethod",$tw.Wiki.prototype);
+	$tw.wiki.addIndexersToWiki();
 	$tw.modules.applyMethods("tiddlerdeserializer",$tw.Wiki.tiddlerDeserializerModules);
 	$tw.macros = $tw.modules.getModulesByTypeAsHashmap("macro");
 	$tw.wiki.initParsers();

core/modules/wiki.js

@@ -27,6 +27,16 @@ var widget = require("$:/core/modules/widgets/widget.js");
 var USER_NAME_TITLE = "$:/status/UserName",
 	TIMESTAMP_DISABLE_TITLE = "$:/config/TimestampDisable";
 
+/*
+Add available indexers to this wiki
+*/
+exports.addIndexersToWiki = function() {
+	var self = this;
+	$tw.utils.each($tw.modules.applyMethods("indexer"),function(Indexer,name) {
+		self.addIndexer(new Indexer(self),name);
+	});
+};
+
 /*
 Get the value of a text reference. Text references can have any of these forms:
 	<tiddlertitle>
@@ -478,11 +488,18 @@ exports.getOrphanTitles = function() {
 Retrieves a list of the tiddler titles that are tagged with a given tag
 */
 exports.getTiddlersWithTag = function(tag) {
-	var self = this;
+	// Try to use the indexer
-	return this.getGlobalCache("taglist-" + tag,function() {
+	var self = this,
+		tagIndexer = this.getIndexer("TagIndexer"),
+		results = tagIndexer && tagIndexer.lookup(tag);
+	if(!results) {
+		// If not available, perform a manual scan
+		results = this.getGlobalCache("taglist-" + tag,function() {
 			var tagmap = self.getTagMap();
 			return self.sortByList(tagmap[tag],tag);
 		});
+	}
+	return results;
 };
 
 /*

editions/test/tiddlers/tests/test-filters.js

@@ -33,9 +33,20 @@ describe("Filter tests", function() {
 		);
 	});
 
+	describe("With no indexers", function() {
+		var wiki = setupWiki({enableIndexers: []});
+		runTests(wiki);
+	});
 
+	describe("With all indexers", function() {
+		var wiki = setupWiki();
+		wiki.getIndexer("FieldIndexer").setMaxIndexedValueLength(8); // Note that JoeBloggs is 9, and JohnDoe is 7
+		runTests(wiki);
+	});
+
+function setupWiki(wikiOptions) {
 	// Create a wiki
-	var wiki = new $tw.Wiki({
+	var wiki = new $tw.Wiki($tw.utils.extend({
 		shadowTiddlers: {
 			"$:/TiddlerFive": {
 				tiddler: new $tw.Tiddler({title: "$:/TiddlerFive",
@@ -64,8 +75,8 @@ describe("Filter tests", function() {
 				})
 			}
 		}
-	});
+	},
-
+	wikiOptions));
 	// Add a few  tiddlers
 	wiki.addTiddler({
 		title: "TiddlerOne",
@@ -99,8 +110,11 @@ describe("Filter tests", function() {
 		list: "[[Tiddler Three]] [[TiddlerOne]]",
 		empty: "",
 		modifier: "JohnDoe"});
+	return wiki;
+}
 
-	// Our tests
+// Our tests
+function runTests(wiki) {
 
 	it("should handle the ~ prefix", function() {
 		expect(wiki.filterTiddlers("[modifier[JoeBloggs]] ~[[No such tiddler]]").join(",")).toBe("TiddlerOne");
@@ -131,6 +145,12 @@ describe("Filter tests", function() {
 		expect(wiki.filterTiddlers("[field:modifier[JoeBloggs]]").join(",")).toBe("TiddlerOne");
 		expect(wiki.filterTiddlers("[!field:modifier[JoeBloggs]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
 		expect(wiki.filterTiddlers("[!is[system]!field:modifier[JoeBloggs]]").join(",")).toBe("Tiddler Three,a fourth tiddler,one");
+		expect(wiki.filterTiddlers("[modifier[JohnDoe]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
+		expect(wiki.filterTiddlers("[!modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
+		expect(wiki.filterTiddlers("[!is[system]!modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
+		expect(wiki.filterTiddlers("[field:modifier[JohnDoe]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
+		expect(wiki.filterTiddlers("[!field:modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
+		expect(wiki.filterTiddlers("[!is[system]!field:modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
 	});
 
 	it("should handle the regexp operator", function() {
@@ -342,7 +362,7 @@ describe("Filter tests", function() {
 			type: "widget",
 			children: [{type: "widget", children: []}]
 		},{
-			wiki: $tw.wiki,
+			wiki: wiki,
 			document: $tw.document
 		});
 		rootWidget.makeChildWidgets();
@@ -388,6 +408,8 @@ describe("Filter tests", function() {
 		expect(wiki.filterTiddlers("1 2 3 4 +[min[2]]").join(",")).toBe("1,2,2,2");
 	});
 
+}
+
 });
 
 })();

editions/test/tiddlers/tests/test-tags.js

@@ -14,8 +14,21 @@ Tests the tagging mechanism.
 
 describe("Tag tests", function() {
 
+describe("With no indexers", function() {
+	var wikiOptions = {enableIndexers: []},
+		wiki = setupWiki(wikiOptions);
+	runTests(wiki,wikiOptions);
+});
+
+describe("With all indexers", function() {
+	var wikiOptions = {enableIndexers: []},
+		wiki = setupWiki();
+	runTests(wiki,wikiOptions);
+});
+
+function setupWiki(wikiOptions) {
 	// Create a wiki
-	var wiki = new $tw.Wiki();
+	var wiki = new $tw.Wiki(wikiOptions);
 
 	// Add a few  tiddlers
 	wiki.addTiddler({
@@ -79,8 +92,11 @@ describe("Tag tests", function() {
 		text: "Another tiddler",
 		tags: ["TiddlerSeventh"],
 		"list-after": "Tiddler Three"});
+	return wiki;
+}
 
 	// Our tests
+function runTests(wiki,wikiOptions) {
 
 	it("should handle custom tag ordering", function() {
 		expect(wiki.filterTiddlers("[tag[TiddlerSeventh]]").join(",")).toBe("Tiddler10,TiddlerOne,Tiddler Three,Tiddler11,Tiddler9,a fourth tiddler");
@@ -88,7 +104,7 @@ describe("Tag tests", function() {
 
 	// Tests for issue (#3296)
 	it("should apply tag ordering in order of dependency", function () {
-		var wiki = new $tw.Wiki();
+		var wiki = new $tw.Wiki(wikiOptions);
 
 		wiki.addTiddler({ title: "A", text: "", tags: "sortTag", "list-after": "B"});
 		wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "C"});
@@ -98,7 +114,7 @@ describe("Tag tests", function() {
 	});
 
 	it("should handle self-referencing dependency without looping infinitely", function() {
-		var wiki = new $tw.Wiki();
+		var wiki = new $tw.Wiki(wikiOptions);
 
 		wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
 		wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "B"});
@@ -108,7 +124,7 @@ describe("Tag tests", function() {
 	});
 
 	it("should handle empty list-after ordering", function() {
-		var wiki = new $tw.Wiki();
+		var wiki = new $tw.Wiki(wikiOptions);
 
 		wiki.addTiddler({ title: "A", text: "", tags: "sortTag", "list-after": ""});
 		wiki.addTiddler({ title: "B", text: "", tags: "sortTag"});
@@ -121,7 +137,7 @@ describe("Tag tests", function() {
 	// with list-after/before, we need to make sure we don't accidentally
 	// handle that external tiddler, or that reference.
 	it("should gracefully handle dependencies that aren't in the tag list", function() {
-		var wiki = new $tw.Wiki();
+		var wiki = new $tw.Wiki(wikiOptions);
 
 		wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
 		wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "Z"});
@@ -132,13 +148,16 @@ describe("Tag tests", function() {
 	});
 
 	it("should handle javascript-specific titles", function() {
-		var wiki = new $tw.Wiki();
+		var wiki = new $tw.Wiki(wikiOptions);
 
 		wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
 		wiki.addTiddler({ title: "__proto__", text: "", tags: "sortTag", "list-before": ""});
 
 		expect(wiki.filterTiddlers("[tag[sortTag]]").join(',')).toBe("__proto__,A");
 	});
+
+}
+
 });
 
 })();