1
0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-06-18 11:29:55 +00:00

Add indexes to the wiki store to improve performance (#3951)

* First pass at modular wiki indexes

An exploratory experiment

* Fix tests

* Faster checking for existence of index methods

We don't really need to check the type

* Use the index for the has operator

* Fix typo

* Move iterator index methods into indexer modules

Now boot.js doesn't know the core indexers

* Fix up the other iterator index functions

* Fix crash with missing index branch

* Limit the field indexer to values less than 128 characters

* Fallback to the old manual scan if the index method returns null
* Sadly, we can no longe re-use the field indexer to accelerate the `has` operator, because the index now omits tiddlers that have field values longer than the limit

Still need to make the index configuration exposed somehow

* Rearrange tests so that we can test with and without indexers

We also need to expose the list of enabled indexers as a config option

* Test the field indexer with different length fields

So that we test the indexed and non-indexed codepaths
This commit is contained in:
Jeremy Ruston 2019-05-24 21:07:37 +01:00 committed by GitHub
parent b992b79adb
commit a8f70b08a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 403 additions and 27 deletions

View File

@ -1035,7 +1035,23 @@ $tw.Wiki = function(options) {
shadowTiddlerTitles = Object.keys(shadowTiddlers);
}
return shadowTiddlerTitles;
};
},
enableIndexers = options.enableIndexers || null, // Array of indexer names to enable, or null to use all available indexers
indexers = [],
indexersByName = Object.create(null);
this.addIndexer = function(indexer,name) {
// Bail if this indexer is not enabled
if(enableIndexers && enableIndexers.indexOf(name) === -1) {
return;
}
indexers.push(indexer);
indexersByName[name] = indexer;
};
this.getIndexer = function(name) {
return indexersByName[name] || null;
};
// Add a tiddler to the store
this.addTiddler = function(tiddler) {
@ -1046,6 +1062,7 @@ $tw.Wiki = function(options) {
if(tiddler) {
var title = tiddler.fields.title;
if(title) {
var oldTiddler = this.getTiddler(title);
// Uncomment the following line for detailed logs of all tiddler writes
// console.log("Adding",title,tiddler)
tiddlers[title] = tiddler;
@ -1054,6 +1071,9 @@ $tw.Wiki = function(options) {
}
this.clearCache(title);
this.clearGlobalCache();
$tw.utils.each(indexers,function(indexer) {
indexer.update(oldTiddler,tiddler);
});
this.enqueueTiddlerEvent(title);
}
}
@ -1064,6 +1084,7 @@ $tw.Wiki = function(options) {
// Uncomment the following line for detailed logs of all tiddler deletions
// console.log("Deleting",title)
if($tw.utils.hop(tiddlers,title)) {
var oldTiddler = this.getTiddler(title);
delete tiddlers[title];
if(tiddlerTitles) {
var index = tiddlerTitles.indexOf(title);
@ -1073,6 +1094,10 @@ $tw.Wiki = function(options) {
}
this.clearCache(title);
this.clearGlobalCache();
var newTiddler = this.getTiddler(title);
$tw.utils.each(indexers,function(indexer) {
indexer.update(oldTiddler,newTiddler);
});
this.enqueueTiddlerEvent(title,true);
}
};
@ -1159,7 +1184,6 @@ $tw.Wiki = function(options) {
callback(tiddlers[title],title);
}
}
};
// Test for the existence of a tiddler (excludes shadow tiddlers)
@ -1273,8 +1297,14 @@ $tw.Wiki = function(options) {
shadowTiddlerTitles = null;
this.clearCache(null);
this.clearGlobalCache();
$tw.utils.each(indexers,function(indexer) {
indexer.rebuild();
});
};
if(this.addIndexersToWiki) {
this.addIndexersToWiki();
}
};
// Dummy methods that will be filled in after boot

View File

@ -16,7 +16,7 @@ Filter operator for comparing fields for equality
Export our filter function
*/
exports.field = function(source,operator,options) {
var results = [],
var results = [],indexedResults,
fieldname = (operator.suffix || operator.operator || "title").toLowerCase();
if(operator.prefix === "!") {
if(operator.regexp) {
@ -53,6 +53,12 @@ exports.field = function(source,operator,options) {
}
});
} else {
if(source.byField) {
indexedResults = source.byField(fieldname,operator.operand);
if(indexedResults) {
return indexedResults
}
}
source(function(tiddler,title) {
if(tiddler) {
var text = tiddler.getFieldString(fieldname);

View File

@ -45,7 +45,7 @@ exports.has = function(source,operator,options) {
if(tiddler && $tw.utils.hop(tiddler.fields,operator.operand) && !(tiddler.fields[operator.operand] === "" || tiddler.fields[operator.operand].length === 0)) {
results.push(title);
}
});
});
}
}
return results;

View File

@ -16,7 +16,7 @@ Filter operator for checking for the presence of a tag
Export our filter function
*/
exports.tag = function(source,operator,options) {
var results = [];
var results = [],indexedResults;
if((operator.suffix || "").toLowerCase() === "strict" && !operator.operand) {
// New semantics:
// Always return copy of input if operator.operand is missing
@ -25,9 +25,10 @@ exports.tag = function(source,operator,options) {
});
} else {
// Old semantics:
var tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
var tiddlers;
if(operator.prefix === "!") {
// Returns a copy of the input if operator.operand is missing
tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
source(function(tiddler,title) {
if(tiddlers.indexOf(title) === -1) {
results.push(title);
@ -35,12 +36,20 @@ exports.tag = function(source,operator,options) {
});
} else {
// Returns empty results if operator.operand is missing
source(function(tiddler,title) {
if(tiddlers.indexOf(title) !== -1) {
results.push(title);
if(source.byTag) {
indexedResults = source.byTag(operator.operand);
if(indexedResults) {
return indexedResults;
}
});
results = options.wiki.sortByList(results,operator.operand);
} else {
tiddlers = options.wiki.getTiddlersWithTag(operator.operand);
source(function(tiddler,title) {
if(tiddlers.indexOf(title) !== -1) {
results.push(title);
}
});
results = options.wiki.sortByList(results,operator.operand);
}
}
}
return results;

View File

@ -0,0 +1,141 @@
/*\
title: $:/core/modules/indexers/field-indexer.js
type: application/javascript
module-type: indexer
Indexes the tiddlers with each field value
\*/
(function(){
/*jslint node: true, browser: true */
/*global modules: false */
"use strict";
var DEFAULT_MAXIMUM_INDEXED_VALUE_LENGTH = 128;
function FieldIndexer(wiki) {
this.wiki = wiki;
this.index = null;
this.maxIndexedValueLength = DEFAULT_MAXIMUM_INDEXED_VALUE_LENGTH;
this.addIndexMethods();
}
// Provided for testing
FieldIndexer.prototype.setMaxIndexedValueLength = function(length) {
this.index = null;
this.maxIndexedValueLength = length;
};
FieldIndexer.prototype.addIndexMethods = function() {
var self = this;
this.wiki.each.byField = function(name,value) {
var titles = self.wiki.allTitles(),
lookup = self.lookup(name,value);
return lookup && lookup.filter(function(title) {
return titles.indexOf(title) !== -1;
});
};
this.wiki.eachShadow.byField = function(name,value) {
var titles = self.wiki.allShadowTitles(),
lookup = self.lookup(name,value);
return lookup && lookup.filter(function(title) {
return titles.indexOf(title) !== -1;
});
};
this.wiki.eachTiddlerPlusShadows.byField = function(name,value) {
var lookup = self.lookup(name,value);
return lookup ? lookup.slice(0) : null;
};
this.wiki.eachShadowPlusTiddlers.byField = function(name,value) {
var lookup = self.lookup(name,value);
return lookup ? lookup.slice(0) : null;
};
};
/*
Tear down and then rebuild the index as if all tiddlers have changed
*/
FieldIndexer.prototype.rebuild = function() {
// Invalidate the index so that it will be rebuilt when it is next used
this.index = null;
};
/*
Build the index for a particular field
*/
FieldIndexer.prototype.buildIndexForField = function(name) {
var self = this;
// Hashmap by field name of hashmap by field value of array of tiddler titles
this.index = this.index || Object.create(null);
this.index[name] = Object.create(null);
var baseIndex = this.index[name];
// Update the index for each tiddler
this.wiki.eachTiddlerPlusShadows(function(tiddler,title) {
if(name in tiddler.fields) {
var value = tiddler.getFieldString(name);
// Skip any values above the maximum length
if(value.length < self.maxIndexedValueLength) {
baseIndex[value] = baseIndex[value] || [];
baseIndex[value].push(title);
}
}
});
};
/*
Update the index in the light of a tiddler value changing; note that the title must be identical. (Renames are handled as a separate delete and create)
oldTiddler: old tiddler value, or null for creation
newTiddler: new tiddler value, or null for deletion
*/
FieldIndexer.prototype.update = function(oldTiddler,newTiddler) {
var self = this;
// Don't do anything if the index hasn't been built yet
if(this.index === null) {
return;
}
// Remove the old tiddler from the index
if(oldTiddler) {
$tw.utils.each(this.index,function(indexEntry,name) {
if(name in oldTiddler.fields) {
var value = oldTiddler.getFieldString(name),
tiddlerList = indexEntry[value];
if(tiddlerList) {
var index = tiddlerList.indexOf(oldTiddler.fields.title);
if(index !== -1) {
tiddlerList.splice(index,1);
}
}
}
});
}
// Add the new tiddler to the index
if(newTiddler) {
$tw.utils.each(this.index,function(indexEntry,name) {
if(name in newTiddler.fields) {
var value = newTiddler.getFieldString(name);
if(value.length < self.maxIndexedValueLength) {
indexEntry[value] = indexEntry[value] || [];
indexEntry[value].push(newTiddler.fields.title);
}
}
});
}
};
// Lookup the given field returning a list of tiddler titles
FieldIndexer.prototype.lookup = function(name,value) {
// Fail the lookup if the value is too long
if(value.length >= this.maxIndexedValueLength) {
return null;
}
// Update the index if it has yet to be built
if(this.index === null || !this.index[name]) {
this.buildIndexForField(name);
}
return this.index[name][value] || [];
};
exports.FieldIndexer = FieldIndexer;
})();

View File

@ -0,0 +1,131 @@
/*\
title: $:/core/modules/indexers/tag-indexer.js
type: application/javascript
module-type: indexer
Indexes the tiddlers with each tag
\*/
(function(){
/*jslint node: true, browser: true */
/*global modules: false */
"use strict";
function TagIndexer(wiki) {
this.wiki = wiki;
this.index = null;
this.addIndexMethods();
}
TagIndexer.prototype.addIndexMethods = function() {
var self = this;
this.wiki.each.byTag = function(tag) {
var titles = self.wiki.allTitles();
return self.lookup(tag).filter(function(title) {
return titles.indexOf(title) !== -1;
});
};
this.wiki.eachShadow.byTag = function(tag) {
var titles = self.wiki.allShadowTitles();
return self.lookup(tag).filter(function(title) {
return titles.indexOf(title) !== -1;
});
};
this.wiki.eachTiddlerPlusShadows.byTag = function(tag) {
return self.lookup(tag).slice(0);
};
this.wiki.eachShadowPlusTiddlers.byTag = function(tag) {
return self.lookup(tag).slice(0);
};
};
/*
Tear down and then rebuild the index as if all tiddlers have changed
*/
TagIndexer.prototype.rebuild = function() {
var self = this;
// Hashmap by tag of array of {isSorted:, titles:[]}
this.index = Object.create(null);
// Add all the tags
this.wiki.eachTiddlerPlusShadows(function(tiddler,title) {
$tw.utils.each(tiddler.fields.tags,function(tag) {
if(!self.index[tag]) {
self.index[tag] = {isSorted: false, titles: [title]};
} else {
self.index[tag].titles.push(title);
}
});
});
};
/*
Update the index in the light of a tiddler value changing; note that the title must be identical. (Renames are handled as a separate delete and create)
oldTiddler: old tiddler value, or null for creation
newTiddler: new tiddler value, or null for deletion
*/
TagIndexer.prototype.update = function(oldTiddler,newTiddler) {
// Don't update the index if it has yet to be built
if(this.index === null) {
return;
}
var self = this,
title = oldTiddler ? oldTiddler.fields.title : newTiddler.fields.title;
// Handle changes to the tags
var oldTiddlerTags = (oldTiddler ? (oldTiddler.fields.tags || []) : []),
newTiddlerTags = (newTiddler ? (newTiddler.fields.tags || []) : []);
$tw.utils.each(oldTiddlerTags,function(oldTag) {
if(newTiddlerTags.indexOf(oldTag) === -1) {
// Deleted tag
var indexRecord = self.index[oldTag],
pos = indexRecord.titles.indexOf(title);
if(pos !== -1) {
indexRecord.titles.splice(pos,1);
}
}
});
$tw.utils.each(newTiddlerTags,function(newTag) {
if(oldTiddlerTags.indexOf(newTag) === -1) {
// New tag
var indexRecord = self.index[newTag];
if(!indexRecord) {
self.index[newTag] = {isSorted: false, titles: [title]};
} else {
indexRecord.titles.push(title);
indexRecord.isSorted = false;
}
}
});
// Handle changes to the list field of tags
var oldTiddlerList = (oldTiddler ? (oldTiddler.fields.list || []) : []),
newTiddlerList = (newTiddler ? (newTiddler.fields.list || []) : []);
if(!$tw.utils.isArrayEqual(oldTiddlerList,newTiddlerList)) {
if(self.index[title]) {
self.index[title].isSorted = false;
}
}
};
// Lookup the given tag returning an ordered list of tiddler titles
TagIndexer.prototype.lookup = function(tag) {
// Update the index if it has yet to be built
if(this.index === null) {
this.rebuild();
}
var indexRecord = this.index[tag];
if(indexRecord) {
if(!indexRecord.isSorted) {
if(this.wiki.sortByList) {
indexRecord.titles = this.wiki.sortByList(indexRecord.titles,tag);
}
indexRecord.isSorted = true;
}
return indexRecord.titles;
} else {
return [];
}
};
exports.TagIndexer = TagIndexer;
})();

View File

@ -27,6 +27,7 @@ exports.startup = function() {
$tw.Tiddler.fieldModules = $tw.modules.getModulesByTypeAsHashmap("tiddlerfield");
$tw.modules.applyMethods("tiddlermethod",$tw.Tiddler.prototype);
$tw.modules.applyMethods("wikimethod",$tw.Wiki.prototype);
$tw.wiki.addIndexersToWiki();
$tw.modules.applyMethods("tiddlerdeserializer",$tw.Wiki.tiddlerDeserializerModules);
$tw.macros = $tw.modules.getModulesByTypeAsHashmap("macro");
$tw.wiki.initParsers();

View File

@ -27,6 +27,16 @@ var widget = require("$:/core/modules/widgets/widget.js");
var USER_NAME_TITLE = "$:/status/UserName",
TIMESTAMP_DISABLE_TITLE = "$:/config/TimestampDisable";
/*
Add available indexers to this wiki
*/
exports.addIndexersToWiki = function() {
var self = this;
$tw.utils.each($tw.modules.applyMethods("indexer"),function(Indexer,name) {
self.addIndexer(new Indexer(self),name);
});
};
/*
Get the value of a text reference. Text references can have any of these forms:
<tiddlertitle>
@ -478,11 +488,18 @@ exports.getOrphanTitles = function() {
Retrieves a list of the tiddler titles that are tagged with a given tag
*/
exports.getTiddlersWithTag = function(tag) {
var self = this;
return this.getGlobalCache("taglist-" + tag,function() {
var tagmap = self.getTagMap();
return self.sortByList(tagmap[tag],tag);
});
// Try to use the indexer
var self = this,
tagIndexer = this.getIndexer("TagIndexer"),
results = tagIndexer && tagIndexer.lookup(tag);
if(!results) {
// If not available, perform a manual scan
results = this.getGlobalCache("taglist-" + tag,function() {
var tagmap = self.getTagMap();
return self.sortByList(tagmap[tag],tag);
});
}
return results;
};
/*

View File

@ -33,9 +33,20 @@ describe("Filter tests", function() {
);
});
describe("With no indexers", function() {
var wiki = setupWiki({enableIndexers: []});
runTests(wiki);
});
describe("With all indexers", function() {
var wiki = setupWiki();
wiki.getIndexer("FieldIndexer").setMaxIndexedValueLength(8); // Note that JoeBloggs is 9, and JohnDoe is 7
runTests(wiki);
});
function setupWiki(wikiOptions) {
// Create a wiki
var wiki = new $tw.Wiki({
var wiki = new $tw.Wiki($tw.utils.extend({
shadowTiddlers: {
"$:/TiddlerFive": {
tiddler: new $tw.Tiddler({title: "$:/TiddlerFive",
@ -64,8 +75,8 @@ describe("Filter tests", function() {
})
}
}
});
},
wikiOptions));
// Add a few tiddlers
wiki.addTiddler({
title: "TiddlerOne",
@ -99,8 +110,11 @@ describe("Filter tests", function() {
list: "[[Tiddler Three]] [[TiddlerOne]]",
empty: "",
modifier: "JohnDoe"});
return wiki;
}
// Our tests
// Our tests
function runTests(wiki) {
it("should handle the ~ prefix", function() {
expect(wiki.filterTiddlers("[modifier[JoeBloggs]] ~[[No such tiddler]]").join(",")).toBe("TiddlerOne");
@ -131,6 +145,12 @@ describe("Filter tests", function() {
expect(wiki.filterTiddlers("[field:modifier[JoeBloggs]]").join(",")).toBe("TiddlerOne");
expect(wiki.filterTiddlers("[!field:modifier[JoeBloggs]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
expect(wiki.filterTiddlers("[!is[system]!field:modifier[JoeBloggs]]").join(",")).toBe("Tiddler Three,a fourth tiddler,one");
expect(wiki.filterTiddlers("[modifier[JohnDoe]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
expect(wiki.filterTiddlers("[!modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
expect(wiki.filterTiddlers("[!is[system]!modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
expect(wiki.filterTiddlers("[field:modifier[JohnDoe]]").join(",")).toBe("$:/TiddlerTwo,Tiddler Three,a fourth tiddler,one");
expect(wiki.filterTiddlers("[!field:modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
expect(wiki.filterTiddlers("[!is[system]!field:modifier[JohnDoe]]").join(",")).toBe("TiddlerOne");
});
it("should handle the regexp operator", function() {
@ -342,7 +362,7 @@ describe("Filter tests", function() {
type: "widget",
children: [{type: "widget", children: []}]
},{
wiki: $tw.wiki,
wiki: wiki,
document: $tw.document
});
rootWidget.makeChildWidgets();
@ -388,6 +408,8 @@ describe("Filter tests", function() {
expect(wiki.filterTiddlers("1 2 3 4 +[min[2]]").join(",")).toBe("1,2,2,2");
});
}
});
})();

View File

@ -14,8 +14,21 @@ Tests the tagging mechanism.
describe("Tag tests", function() {
describe("With no indexers", function() {
var wikiOptions = {enableIndexers: []},
wiki = setupWiki(wikiOptions);
runTests(wiki,wikiOptions);
});
describe("With all indexers", function() {
var wikiOptions = {enableIndexers: []},
wiki = setupWiki();
runTests(wiki,wikiOptions);
});
function setupWiki(wikiOptions) {
// Create a wiki
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
// Add a few tiddlers
wiki.addTiddler({
@ -79,8 +92,11 @@ describe("Tag tests", function() {
text: "Another tiddler",
tags: ["TiddlerSeventh"],
"list-after": "Tiddler Three"});
return wiki;
}
// Our tests
function runTests(wiki,wikiOptions) {
it("should handle custom tag ordering", function() {
expect(wiki.filterTiddlers("[tag[TiddlerSeventh]]").join(",")).toBe("Tiddler10,TiddlerOne,Tiddler Three,Tiddler11,Tiddler9,a fourth tiddler");
@ -88,7 +104,7 @@ describe("Tag tests", function() {
// Tests for issue (#3296)
it("should apply tag ordering in order of dependency", function () {
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
wiki.addTiddler({ title: "A", text: "", tags: "sortTag", "list-after": "B"});
wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "C"});
@ -98,7 +114,7 @@ describe("Tag tests", function() {
});
it("should handle self-referencing dependency without looping infinitely", function() {
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "B"});
@ -108,7 +124,7 @@ describe("Tag tests", function() {
});
it("should handle empty list-after ordering", function() {
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
wiki.addTiddler({ title: "A", text: "", tags: "sortTag", "list-after": ""});
wiki.addTiddler({ title: "B", text: "", tags: "sortTag"});
@ -121,7 +137,7 @@ describe("Tag tests", function() {
// with list-after/before, we need to make sure we don't accidentally
// handle that external tiddler, or that reference.
it("should gracefully handle dependencies that aren't in the tag list", function() {
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
wiki.addTiddler({ title: "B", text: "", tags: "sortTag", "list-after": "Z"});
@ -132,13 +148,16 @@ describe("Tag tests", function() {
});
it("should handle javascript-specific titles", function() {
var wiki = new $tw.Wiki();
var wiki = new $tw.Wiki(wikiOptions);
wiki.addTiddler({ title: "A", text: "", tags: "sortTag"});
wiki.addTiddler({ title: "__proto__", text: "", tags: "sortTag", "list-before": ""});
expect(wiki.filterTiddlers("[tag[sortTag]]").join(',')).toBe("__proto__,A");
});
}
});
})();