mirrors/TiddlyWiki5
1
0
Fork 0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-11-30 05:19:57 +00:00
Code Issues Releases Wiki Activity

MWS: Add support for node-sqlite-wasm alongside better-sqlite3 (#7996)

Browse Source 
* Switch from better-sqlite3 to node-sqlite3-wasm

Seems to be slower, but might make cloud deployments easier by not having any binary dependencies

* More logging

* Temporarily use a memory database

We will make this configurable

* Revert "More logging"

* Resume loading demo tiddlers

* Cache prepared statements

Gives a 20% reduction in startup time on my machine

* Some more logging

* Update package-lock

* More logging

* Route regexps should allow for proxies that automatically decode URLs

Astonishingly, Azure does this

* Go back to a file-based database

* Less logging

* Update package-lock.json

* Simplify startup by not loading the docs edition

* Tiddler database layer should mark statements as having been removed

* Re-introduce better-sqlite3

* Make the SQLite provider be switchable

* Support switchable SQL engines

I am not intending to make this a long term feature. We will choose one engine and stick with it until we choose to change to another.

* Adjust dependency versions

* Setting up default engine

* Make transaction handling compatible with node-sqlite3-wasm

https://github.com/tndrle/node-sqlite3-wasm doesn't have transaction support so I've tried to implement it using SQL statements directly.

@hoelzro do you think this is right? Should we be rolling back the transaction in the finally clause? It would be nice to have tests in this area...

I looked at better-sqlite3's implementation - https://github.com/WiseLibs/better-sqlite3/blob/master/lib/methods/transaction.js

* Default to better-sqlite3 for compatibility after merging
This commit is contained in:
Jeremy Ruston 2024-02-22 11:57:41 +00:00 committed by GitHub
parent 790f431df0
commit 3fca82321e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 126 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
package-lock.json generated
View File

@ -9,7 +9,8 @@
"version": "5.3.4-prerelease", "version": "5.3.4-prerelease",
"license": "BSD", "license": "BSD",
"dependencies": { "dependencies": {
"better-sqlite3": "^9.4.3" "better-sqlite3": "^9.4.3",
"node-sqlite3-wasm": "^0.8.10"
}, },
"bin": { "bin": {
"tiddlywiki": "tiddlywiki.js" "tiddlywiki": "tiddlywiki.js"
@ -1114,6 +1115,11 @@
"node": ">=10" "node": ">=10"
} }
}, },
"node_modules/node-sqlite3-wasm": {
"version": "0.8.10",
"resolved": "https://registry.npmjs.org/node-sqlite3-wasm/-/node-sqlite3-wasm-0.8.10.tgz",
"integrity": "sha512-is4xaYRCIHxYsL9rY7AbutnPIoOqBqEzIiQrm/X5QzFQEgyF18Rob3Wj096NuShTWq6ZmQIMcvTaF5vfHbSdjw=="
},
"node_modules/once": { "node_modules/once": {
"version": "1.4.0", "version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",

3
package.json
View File

@ -37,6 +37,7 @@
"lint": "eslint ." "lint": "eslint ."
}, },
"dependencies": { "dependencies": {
"better-sqlite3": "^9.4.3" "better-sqlite3": "^9.4.3",
"node-sqlite3-wasm": "^0.8.10"
} }
} }

43
plugins/tiddlywiki/multiwikiserver/modules/init.js
View File

@ -20,21 +20,6 @@ exports.synchronous = true;
exports.startup = function() { exports.startup = function() {
var path = require("path"); var path = require("path");
// Install the sqlite3 global namespace
$tw.sqlite3 = {
Database: null
};
// Check that better-sqlite3 is installed
var logger = new $tw.utils.Logger("multiwikiserver");
try {
$tw.sqlite3.Database = require("better-sqlite3");
} catch(e) {
}
console.log(`Successfully required better-sqlite3`)
if(!$tw.sqlite3.Database) {
logger.alert("The plugin 'tiddlywiki/multiwikiserver' requires the better-sqlite3 npm package to be installed. Run 'npm install' in the root of the TiddlyWiki repository");
return;
}
// Create and initialise the tiddler store and upload manager // Create and initialise the tiddler store and upload manager
var SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore, var SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore,
store = new SqlTiddlerStore({ store = new SqlTiddlerStore({
@ -51,13 +36,27 @@ console.log(`Successfully required better-sqlite3`)
}; };
// Performance timing // Performance timing
console.time("mws-initial-load"); console.time("mws-initial-load");
// Create docs bag and recipe // Copy TiddlyWiki core editions
$tw.mws.store.createBag("docs","TiddlyWiki Documentation from https://tiddlywiki.com/"); function copyEdition(options) {
$tw.mws.store.createRecipe("docs",["docs"],"TiddlyWiki Documentation from https://tiddlywiki.com/"); console.log(`Copying edition ${options.tiddlersPath}`);
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"tw5.com/tiddlers"),"docs"); $tw.mws.store.createBag(options.bagName,options.bagDescription);
$tw.mws.store.createBag("dev-docs","TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/"); $tw.mws.store.createRecipe(options.recipeName,[options.bagName],options.recipeDescription);
$tw.mws.store.createRecipe("dev-docs",["dev-docs"],"TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/"); $tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,options.tiddlersPath),options.bagName);
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"dev/tiddlers"),"dev-docs"); }
// copyEdition({
// bagName: "docs",
// bagDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
// recipeName: "docs",
// recipeDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
// tiddlersPath: "tw5.com/tiddlers"
// });
copyEdition({
bagName: "dev-docs",
bagDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
recipeName: "dev-docs",
recipeDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
tiddlersPath: "dev/tiddlers"
});
// Create bags and recipes // Create bags and recipes
$tw.mws.store.createBag("bag-alpha","A test bag"); $tw.mws.store.createBag("bag-alpha","A test bag");
$tw.mws.store.createBag("bag-beta","Another test bag"); $tw.mws.store.createBag("bag-beta","Another test bag");

128
plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-database.js
View File

@ -16,38 +16,80 @@ Validation is for the most part left to the caller
Create a tiddler store. Options include: Create a tiddler store. Options include:
databasePath - path to the database file (can be ":memory:" to get a temporary database) databasePath - path to the database file (can be ":memory:" to get a temporary database)
engine - wasm | better
*/ */
function SqlTiddlerDatabase(options) { function SqlTiddlerDatabase(options) {
options = options || {}; options = options || {};
// Create the database // Initialise the statement cache
this.statements = Object.create(null); // Hashmap by SQL text of statement objects
// Create the database file directories if needed
if(options.databasePath) { if(options.databasePath) {
$tw.utils.createFileDirectories(options.databasePath); $tw.utils.createFileDirectories(options.databasePath);
} }
var databasePath = options.databasePath || ":memory:"; // Choose engine
this.db = new $tw.sqlite3.Database(databasePath,{verbose: undefined && console.log}); this.engine = options.engine || "better"; // wasm | better
// Create the database
const databasePath = options.databasePath || ":memory:";
let Database;
console.log(`Creating SQL engine ${this.engine}`)
switch(this.engine) {
case "wasm":
({ Database } = require("node-sqlite3-wasm"));
break;
case "better":
Database = require("better-sqlite3");
break;
}
this.db = new Database(databasePath,{
verbose: undefined && console.log
});
this.transactionDepth = 0; this.transactionDepth = 0;
} }
SqlTiddlerDatabase.prototype.close = function() { SqlTiddlerDatabase.prototype.close = function() {
for(const sql in this.statements) {
this.statements[sql].finalize();
}
this.statements = Object.create(null);
this.db.close(); this.db.close();
this.db = undefined; this.db = undefined;
}; };
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) { SqlTiddlerDatabase.prototype.normaliseParams = function(params) {
params = params || {}; params = params || {};
const statement = this.db.prepare(sql); const result = Object.create(null);
for(const paramName in params) {
if(this.engine !== "wasm" && paramName.startsWith("$")) {
result[paramName.slice(1)] = params[paramName];
} else {
result[paramName] = params[paramName];
}
}
return result;
};
SqlTiddlerDatabase.prototype.prepareStatement = function(sql) {
if(!(sql in this.statements)) {
this.statements[sql] = this.db.prepare(sql);
}
return this.statements[sql];
};
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) {
params = this.normaliseParams(params);
const statement = this.prepareStatement(sql);
return statement.run(params); return statement.run(params);
}; };
SqlTiddlerDatabase.prototype.runStatementGet = function(sql,params) { SqlTiddlerDatabase.prototype.runStatementGet = function(sql,params) {
params = params || {}; params = this.normaliseParams(params);
const statement = this.db.prepare(sql); const statement = this.prepareStatement(sql);
return statement.get(params); return statement.get(params);
}; };
SqlTiddlerDatabase.prototype.runStatementGetAll = function(sql,params) { SqlTiddlerDatabase.prototype.runStatementGetAll = function(sql,params) {
params = params || {}; params = this.normaliseParams(params);
const statement = this.db.prepare(sql); const statement = this.prepareStatement(sql);
return statement.all(params); return statement.all(params);
}; };
@ -134,7 +176,7 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
INSERT OR IGNORE INTO bags (bag_name, accesscontrol, description) INSERT OR IGNORE INTO bags (bag_name, accesscontrol, description)
VALUES ($bag_name, '', '') VALUES ($bag_name, '', '')
`,{ `,{
bag_name: bagname $bag_name: bagname
}); });
this.runStatement(` this.runStatement(`
UPDATE bags UPDATE bags
@ -142,9 +184,9 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
description = $description description = $description
WHERE bag_name = $bag_name WHERE bag_name = $bag_name
`,{ `,{
bag_name: bagname, $bag_name: bagname,
accesscontrol: "[some access control stuff]", $accesscontrol: "[some access control stuff]",
description: description $description: description
}); });
}; };
@ -182,15 +224,15 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
-- Delete existing recipe_bags entries for this recipe -- Delete existing recipe_bags entries for this recipe
DELETE FROM recipe_bags WHERE recipe_id = (SELECT recipe_id FROM recipes WHERE recipe_name = $recipe_name) DELETE FROM recipe_bags WHERE recipe_id = (SELECT recipe_id FROM recipes WHERE recipe_name = $recipe_name)
`,{ `,{
recipe_name: recipename $recipe_name: recipename
}); });
this.runStatement(` this.runStatement(`
-- Create the entry in the recipes table if required -- Create the entry in the recipes table if required
INSERT OR REPLACE INTO recipes (recipe_name, description) INSERT OR REPLACE INTO recipes (recipe_name, description)
VALUES ($recipe_name, $description) VALUES ($recipe_name, $description)
`,{ `,{
recipe_name: recipename, $recipe_name: recipename,
description: description $description: description
}); });
this.runStatement(` this.runStatement(`
INSERT INTO recipe_bags (recipe_id, bag_id, position) INSERT INTO recipe_bags (recipe_id, bag_id, position)
@ -200,8 +242,8 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
INNER JOIN json_each($bag_names) AS j ON j.value = b.bag_name INNER JOIN json_each($bag_names) AS j ON j.value = b.bag_name
WHERE r.recipe_name = $recipe_name WHERE r.recipe_name = $recipe_name
`,{ `,{
recipe_name: recipename, $recipe_name: recipename,
bag_names: JSON.stringify(bagnames) $bag_names: JSON.stringify(bagnames)
}); });
}; };
@ -217,8 +259,8 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
$title $title
) )
`,{ `,{
title: tiddlerFields.title, $title: tiddlerFields.title,
bag_name: bagname $bag_name: bagname
}); });
// Update the fields table // Update the fields table
this.runStatement(` this.runStatement(`
@ -238,9 +280,9 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
) AS t ) AS t
JOIN json_each($field_values) AS json_each JOIN json_each($field_values) AS json_each
`,{ `,{
title: tiddlerFields.title, $title: tiddlerFields.title,
bag_name: bagname, $bag_name: bagname,
field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined})) $field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined}))
}); });
return { return {
tiddler_id: info.lastInsertRowid tiddler_id: info.lastInsertRowid
@ -268,7 +310,7 @@ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipena
) AS selected_bag ) AS selected_bag
ON b.bag_id = selected_bag.bag_id ON b.bag_id = selected_bag.bag_id
`,{ `,{
recipe_name: recipename $recipe_name: recipename
}); });
if(!row) { if(!row) {
return null; return null;
@ -292,8 +334,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
WHERE b.bag_name = $bag_name AND t.title = $title WHERE b.bag_name = $bag_name AND t.title = $title
) )
`,{ `,{
title: title, $title: title,
bag_name: bagname $bag_name: bagname
}); });
this.runStatement(` this.runStatement(`
DELETE FROM tiddlers DELETE FROM tiddlers
@ -303,8 +345,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
WHERE bag_name = $bag_name WHERE bag_name = $bag_name
) AND title = $title ) AND title = $title
`,{ `,{
title: title, $title: title,
bag_name: bagname $bag_name: bagname
}); });
}; };
@ -322,8 +364,8 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) {
WHERE t.title = $title AND b.bag_name = $bag_name WHERE t.title = $title AND b.bag_name = $bag_name
) )
`,{ `,{
title: title, $title: title,
bag_name: bagname $bag_name: bagname
}); });
if(rows.length === 0) { if(rows.length === 0) {
return null; return null;
@ -353,8 +395,8 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
ORDER BY rb.position DESC ORDER BY rb.position DESC
LIMIT 1 LIMIT 1
`,{ `,{
title: title, $title: title,
recipe_name: recipename $recipe_name: recipename
}); });
if(!rowTiddlerId) { if(!rowTiddlerId) {
return null; return null;
@ -365,8 +407,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
FROM fields FROM fields
WHERE tiddler_id = $tiddler_id WHERE tiddler_id = $tiddler_id
`,{ `,{
tiddler_id: rowTiddlerId.tiddler_id, $tiddler_id: rowTiddlerId.tiddler_id
recipe_name: recipename
}); });
return { return {
bag_name: rowTiddlerId.bag_name, bag_name: rowTiddlerId.bag_name,
@ -392,7 +433,7 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) {
) )
ORDER BY title ASC ORDER BY title ASC
`,{ `,{
bag_name: bagname $bag_name: bagname
}); });
return rows.map(value => value.title); return rows.map(value => value.title);
}; };
@ -414,7 +455,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) {
ORDER BY t.title ORDER BY t.title
) )
`,{ `,{
recipe_name: recipename $recipe_name: recipename
}); });
return rows; return rows;
}; };
@ -428,7 +469,7 @@ SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bagname) {
WHERE bag_name = $bag_name WHERE bag_name = $bag_name
) )
`,{ `,{
bag_name: bagname $bag_name: bagname
}); });
}; };
@ -448,7 +489,7 @@ SqlTiddlerDatabase.prototype.getRecipeBags = function(recipename) {
) AS bag_priority ON bags.bag_id = bag_priority.bag_id ) AS bag_priority ON bags.bag_id = bag_priority.bag_id
ORDER BY position ORDER BY position
`,{ `,{
recipe_name: recipename $recipe_name: recipename
}); });
return rows.map(value => value.bag_name); return rows.map(value => value.bag_name);
}; };
@ -459,17 +500,22 @@ Execute the given function in a transaction, committing if successful but rollin
Calls to this function can be safely nested, but only the top-most call will actually take place in a transaction. Calls to this function can be safely nested, but only the top-most call will actually take place in a transaction.
*/ */
SqlTiddlerDatabase.prototype.transaction = function(fn) { SqlTiddlerDatabase.prototype.transaction = function(fn) {
try {
const alreadyInTransaction = this.transactionDepth > 0; const alreadyInTransaction = this.transactionDepth > 0;
this.transactionDepth++; this.transactionDepth++;
if(alreadyInTransaction) { if(alreadyInTransaction) {
return fn(); return fn();
} else { } else {
return this.db.transaction(fn)(); try {
} this.runStatement(`BEGIN TRANSACTION`);
var result = fn();
this.runStatement(`COMMIT TRANSACTION`);
} catch(e) {
this.runStatement(`ROLLBACK TRANSACTION`);
} finally { } finally {
this.transactionDepth--; this.transactionDepth--;
} }
return result;
}
}; };
exports.SqlTiddlerDatabase = SqlTiddlerDatabase; exports.SqlTiddlerDatabase = SqlTiddlerDatabase;

4
plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-store.js
View File

@ -20,6 +20,7 @@ Create a tiddler store. Options include:
databasePath - path to the database file (can be ":memory:" to get a temporary database) databasePath - path to the database file (can be ":memory:" to get a temporary database)
adminWiki - reference to $tw.Wiki object into which entity state tiddlers should be saved adminWiki - reference to $tw.Wiki object into which entity state tiddlers should be saved
engine - wasm | better
*/ */
function SqlTiddlerStore(options) { function SqlTiddlerStore(options) {
options = options || {}; options = options || {};
@ -29,7 +30,8 @@ function SqlTiddlerStore(options) {
this.databasePath = options.databasePath || ":memory:"; this.databasePath = options.databasePath || ":memory:";
var SqlTiddlerDatabase = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-database.js").SqlTiddlerDatabase; var SqlTiddlerDatabase = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-database.js").SqlTiddlerDatabase;
this.sqlTiddlerDatabase = new SqlTiddlerDatabase({ this.sqlTiddlerDatabase = new SqlTiddlerDatabase({
databasePath: this.databasePath databasePath: this.databasePath,
engine: options.engine
}); });
this.sqlTiddlerDatabase.createTables(); this.sqlTiddlerDatabase.createTables();
this.updateAdminWiki(); this.updateAdminWiki();