mirrors/TiddlyWiki5
1
0
Fork 0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-11-05 01:26:18 +00:00
Code Issues Releases Wiki Activity

Experiment with optimising specific filters with direct SQL equivalents

Browse Source
This commit is contained in:
Jeremy Ruston 2023-07-21 13:21:04 +01:00
parent b9245da91f
commit 39d04517dd
3 changed files with 94 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
core/modules/filters.js
View File

@ -225,6 +225,8 @@ source: an iterator function for the source tiddlers, called source(iterator), w
widget: an optional widget node for retrieving the current tiddler etc.
*/
exports.compileFilter = function(filterString) {
var self = this;
// Use cached filter function if already present
if(!this.filterCache) {
this.filterCache = Object.create(null);
this.filterCacheCount = 0;
@ -232,21 +234,64 @@ exports.compileFilter = function(filterString) {
if(this.filterCache[filterString] !== undefined) {
return this.filterCache[filterString];
}
var filterParseTree;
try {
filterParseTree = this.parseFilter(filterString);
} catch(e) {
// We do not cache this result, so it adjusts along with localization changes
return function(source,widget) {
return [$tw.language.getString("Error/Filter") + ": " + e];
};
// Attempt to optimise the filter into a single query
var operationFunctions = this.optimiseFilter && this.optimiseFilter(filterString);
// Otherwise compile the filter step by step
if(!operationFunctions) {
// Parse filter
var filterParseTree;
try {
filterParseTree = this.parseFilter(filterString);
} catch(e) {
// We do not cache this result, so it adjusts along with localization changes
return function(source,widget) {
return [$tw.language.getString("Error/Filter") + ": " + e];
};
}
// Compile the filter operators into functions
operationFunctions = this.compileFilterOperations(filterParseTree);
}
// Return a function that applies the operations to a source iterator of tiddler titles
var fnMeasured = $tw.perf.measure("filter: " + filterString,function filterFunction(source,widget) {
if(!source) {
source = self.each;
} else if(typeof source === "object") { // Array or hashmap
source = self.makeTiddlerIterator(source);
}
if(!widget) {
widget = $tw.rootWidget;
}
var results = new $tw.utils.LinkedList();
self.filterRecursionCount = (self.filterRecursionCount || 0) + 1;
if(self.filterRecursionCount < MAX_FILTER_DEPTH) {
$tw.utils.each(operationFunctions,function(operationFunction) {
operationFunction(results,source,widget);
});
} else {
results.push("/**-- Excessive filter recursion --**/");
}
self.filterRecursionCount = self.filterRecursionCount - 1;
return results.toArray();
});
if(this.filterCacheCount >= 2000) {
// To prevent memory leak, we maintain an upper limit for cache size.
// Reset if exceeded. This should give us 95% of the benefit
// that no cache limit would give us.
this.filterCache = Object.create(null);
this.filterCacheCount = 0;
}
this.filterCache[filterString] = fnMeasured;
this.filterCacheCount++;
return fnMeasured;
};
exports.compileFilterOperations = function(filterParseTree) {
var self = this;
// Get the hashmap of filter operator functions
var filterOperators = this.getFilterOperators();
// Assemble array of functions, one for each operation
var operationFunctions = [];
// Step through the operations
var self = this;
$tw.utils.each(filterParseTree,function(operation) {
// Create a function for the chain of operators in the operation
var operationSubFunction = function(source,widget) {
@ -334,38 +379,7 @@ exports.compileFilter = function(filterString) {
}
})());
});
// Return a function that applies the operations to a source iterator of tiddler titles
var fnMeasured = $tw.perf.measure("filter: " + filterString,function filterFunction(source,widget) {
if(!source) {
source = self.each;
} else if(typeof source === "object") { // Array or hashmap
source = self.makeTiddlerIterator(source);
}
if(!widget) {
widget = $tw.rootWidget;
}
var results = new $tw.utils.LinkedList();
self.filterRecursionCount = (self.filterRecursionCount || 0) + 1;
if(self.filterRecursionCount < MAX_FILTER_DEPTH) {
$tw.utils.each(operationFunctions,function(operationFunction) {
operationFunction(results,source,widget);
});
} else {
results.push("/**-- Excessive filter recursion --**/");
}
self.filterRecursionCount = self.filterRecursionCount - 1;
return results.toArray();
});
if(this.filterCacheCount >= 2000) {
// To prevent memory leak, we maintain an upper limit for cache size.
// Reset if exceeded. This should give us 95% of the benefit
// that no cache limit would give us.
this.filterCache = Object.create(null);
this.filterCacheCount = 0;
}
this.filterCache[filterString] = fnMeasured;
this.filterCacheCount++;
return fnMeasured;
return operationFunctions;
};
})();

30
plugins/tiddlywiki/sqlite3store/sql-functions.js
View File

@ -319,6 +319,36 @@ $tw.SqlFunctions = function(options) {
return resultRows;
};
/*
An optimisation of the filter [all[shadows+tiddlers]prefix[$:/language/Docs/Types/]get[name]length[]maxall[]]
*/
var statementQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength = self.db.prepare(`
SELECT MAX(LENGTH(name)) AS max_length
FROM (
SELECT title, shadow, JSON_EXTRACT(meta, '$.name') AS name
FROM tiddlers
WHERE title LIKE '$:/language/Docs/Types/%'
AND (shadow = 0 OR (shadow = 1 AND NOT EXISTS (
SELECT 1
FROM tiddlers AS t2
WHERE t2.title = tiddlers.title
AND t2.shadow = 0
)))
);
`);
this.sqlQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength = function() {
// We return a filter operation function that actually performs the query
return function(results,source,widget) {
var result = 0;
if(statementQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength.step()) {
var row = statementQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength.get({});
result = row.max_length;
}
statementQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength.reset();
results.clear();
results.push(result.toString());
};
};
/*
Debugging
*/
var statementLogTiddlerTable = self.db.prepare("select title, shadow, meta, text from tiddlers order by title,shadow;"),

9
plugins/tiddlywiki/sqlite3store/sql-wiki-store.js
View File

@ -371,6 +371,15 @@ $tw.Wiki = function(options) {
}
};
this.optimiseFilter = function(filterString) {
switch($tw.utils.trim(filterString)) {
case "[all[shadows+tiddlers]prefix[$:/language/Docs/Types/]get[name]length[]maxall[]]":
return [this.sqlFunctions.sqlQuickFilterAllShadowsTiddlersPrefixDocTypeMaxLength()];
break;
}
return undefined;
};
if(this.addIndexersToWiki) {
this.addIndexersToWiki();
}