mirrors/TiddlyWiki5
1
0
Fork 0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-11-27 03:57:21 +00:00
Code Issues Releases Wiki Activity

MWS: Add support for node-sqlite-wasm alongside better-sqlite3 (#7996)

Browse Source 
* Switch from better-sqlite3 to node-sqlite3-wasm

Seems to be slower, but might make cloud deployments easier by not having any binary dependencies

* More logging

* Temporarily use a memory database

We will make this configurable

* Revert "More logging"

* Resume loading demo tiddlers

* Cache prepared statements

Gives a 20% reduction in startup time on my machine

* Some more logging

* Update package-lock

* More logging

* Route regexps should allow for proxies that automatically decode URLs

Astonishingly, Azure does this

* Go back to a file-based database

* Less logging

* Update package-lock.json

* Simplify startup by not loading the docs edition

* Tiddler database layer should mark statements as having been removed

* Re-introduce better-sqlite3

* Make the SQLite provider be switchable

* Support switchable SQL engines

I am not intending to make this a long term feature. We will choose one engine and stick with it until we choose to change to another.

* Adjust dependency versions

* Setting up default engine

* Make transaction handling compatible with node-sqlite3-wasm

https://github.com/tndrle/node-sqlite3-wasm doesn't have transaction support so I've tried to implement it using SQL statements directly.

@hoelzro do you think this is right? Should we be rolling back the transaction in the finally clause? It would be nice to have tests in this area...

I looked at better-sqlite3's implementation - https://github.com/WiseLibs/better-sqlite3/blob/master/lib/methods/transaction.js

* Default to better-sqlite3 for compatibility after merging
This commit is contained in:
Jeremy Ruston 2024-02-22 11:57:41 +00:00 committed by GitHub
parent 790f431df0
commit 3fca82321e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 126 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
package-lock.json generated
View File

@ -9,7 +9,8 @@
"version": "5.3.4-prerelease",
"license": "BSD",
"dependencies": {
"better-sqlite3": "^9.4.3"
"better-sqlite3": "^9.4.3",
"node-sqlite3-wasm": "^0.8.10"
},
"bin": {
"tiddlywiki": "tiddlywiki.js"
@ -1114,6 +1115,11 @@
"node": ">=10"
}
},
"node_modules/node-sqlite3-wasm": {
"version": "0.8.10",
"resolved": "https://registry.npmjs.org/node-sqlite3-wasm/-/node-sqlite3-wasm-0.8.10.tgz",
"integrity": "sha512-is4xaYRCIHxYsL9rY7AbutnPIoOqBqEzIiQrm/X5QzFQEgyF18Rob3Wj096NuShTWq6ZmQIMcvTaF5vfHbSdjw=="
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",

3
package.json
View File

@ -37,6 +37,7 @@
"lint": "eslint ."
},
"dependencies": {
"better-sqlite3": "^9.4.3"
"better-sqlite3": "^9.4.3",
"node-sqlite3-wasm": "^0.8.10"
}
}

43
plugins/tiddlywiki/multiwikiserver/modules/init.js
View File

@ -20,21 +20,6 @@ exports.synchronous = true;
exports.startup = function() {
var path = require("path");
// Install the sqlite3 global namespace
$tw.sqlite3 = {
Database: null
};
// Check that better-sqlite3 is installed
var logger = new $tw.utils.Logger("multiwikiserver");
try {
$tw.sqlite3.Database = require("better-sqlite3");
} catch(e) {
}
console.log(`Successfully required better-sqlite3`)
if(!$tw.sqlite3.Database) {
logger.alert("The plugin 'tiddlywiki/multiwikiserver' requires the better-sqlite3 npm package to be installed. Run 'npm install' in the root of the TiddlyWiki repository");
return;
}
// Create and initialise the tiddler store and upload manager
var SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore,
store = new SqlTiddlerStore({
@ -51,13 +36,27 @@ console.log(`Successfully required better-sqlite3`)
};
// Performance timing
console.time("mws-initial-load");
// Create docs bag and recipe
$tw.mws.store.createBag("docs","TiddlyWiki Documentation from https://tiddlywiki.com/");
$tw.mws.store.createRecipe("docs",["docs"],"TiddlyWiki Documentation from https://tiddlywiki.com/");
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"tw5.com/tiddlers"),"docs");
$tw.mws.store.createBag("dev-docs","TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/");
$tw.mws.store.createRecipe("dev-docs",["dev-docs"],"TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/");
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"dev/tiddlers"),"dev-docs");
// Copy TiddlyWiki core editions
function copyEdition(options) {
console.log(`Copying edition ${options.tiddlersPath}`);
$tw.mws.store.createBag(options.bagName,options.bagDescription);
$tw.mws.store.createRecipe(options.recipeName,[options.bagName],options.recipeDescription);
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,options.tiddlersPath),options.bagName);
}
// copyEdition({
// bagName: "docs",
// bagDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
// recipeName: "docs",
// recipeDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
// tiddlersPath: "tw5.com/tiddlers"
// });
copyEdition({
bagName: "dev-docs",
bagDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
recipeName: "dev-docs",
recipeDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
tiddlersPath: "dev/tiddlers"
});
// Create bags and recipes
$tw.mws.store.createBag("bag-alpha","A test bag");
$tw.mws.store.createBag("bag-beta","Another test bag");

140
plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-database.js
View File

@ -16,38 +16,80 @@ Validation is for the most part left to the caller
Create a tiddler store. Options include:
databasePath - path to the database file (can be ":memory:" to get a temporary database)
engine - wasm | better
*/
function SqlTiddlerDatabase(options) {
options = options || {};
// Create the database
// Initialise the statement cache
this.statements = Object.create(null); // Hashmap by SQL text of statement objects
// Create the database file directories if needed
if(options.databasePath) {
$tw.utils.createFileDirectories(options.databasePath);
}
var databasePath = options.databasePath || ":memory:";
this.db = new $tw.sqlite3.Database(databasePath,{verbose: undefined && console.log});
// Choose engine
this.engine = options.engine || "better"; // wasm | better
// Create the database
const databasePath = options.databasePath || ":memory:";
let Database;
console.log(`Creating SQL engine ${this.engine}`)
switch(this.engine) {
case "wasm":
({ Database } = require("node-sqlite3-wasm"));
break;
case "better":
Database = require("better-sqlite3");
break;
}
this.db = new Database(databasePath,{
verbose: undefined && console.log
});
this.transactionDepth = 0;
}
SqlTiddlerDatabase.prototype.close = function() {
for(const sql in this.statements) {
this.statements[sql].finalize();
}
this.statements = Object.create(null);
this.db.close();
this.db = undefined;
};
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) {
SqlTiddlerDatabase.prototype.normaliseParams = function(params) {
params = params || {};
const statement = this.db.prepare(sql);
const result = Object.create(null);
for(const paramName in params) {
if(this.engine !== "wasm" && paramName.startsWith("$")) {
result[paramName.slice(1)] = params[paramName];
} else {
result[paramName] = params[paramName];
}
}
return result;
};
SqlTiddlerDatabase.prototype.prepareStatement = function(sql) {
if(!(sql in this.statements)) {
this.statements[sql] = this.db.prepare(sql);
}
return this.statements[sql];
};
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) {
params = this.normaliseParams(params);
const statement = this.prepareStatement(sql);
return statement.run(params);
};
SqlTiddlerDatabase.prototype.runStatementGet = function(sql,params) {
params = params || {};
const statement = this.db.prepare(sql);
params = this.normaliseParams(params);
const statement = this.prepareStatement(sql);
return statement.get(params);
};
SqlTiddlerDatabase.prototype.runStatementGetAll = function(sql,params) {
params = params || {};
const statement = this.db.prepare(sql);
params = this.normaliseParams(params);
const statement = this.prepareStatement(sql);
return statement.all(params);
};
@ -134,7 +176,7 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
INSERT OR IGNORE INTO bags (bag_name, accesscontrol, description)
VALUES ($bag_name, '', '')
`,{
bag_name: bagname
$bag_name: bagname
});
this.runStatement(`
UPDATE bags
@ -142,9 +184,9 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
description = $description
WHERE bag_name = $bag_name
`,{
bag_name: bagname,
accesscontrol: "[some access control stuff]",
description: description
$bag_name: bagname,
$accesscontrol: "[some access control stuff]",
$description: description
});
};
@ -182,15 +224,15 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
-- Delete existing recipe_bags entries for this recipe
DELETE FROM recipe_bags WHERE recipe_id = (SELECT recipe_id FROM recipes WHERE recipe_name = $recipe_name)
`,{
recipe_name: recipename
$recipe_name: recipename
});
this.runStatement(`
-- Create the entry in the recipes table if required
INSERT OR REPLACE INTO recipes (recipe_name, description)
VALUES ($recipe_name, $description)
`,{
recipe_name: recipename,
description: description
$recipe_name: recipename,
$description: description
});
this.runStatement(`
INSERT INTO recipe_bags (recipe_id, bag_id, position)
@ -200,8 +242,8 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
INNER JOIN json_each($bag_names) AS j ON j.value = b.bag_name
WHERE r.recipe_name = $recipe_name
`,{
recipe_name: recipename,
bag_names: JSON.stringify(bagnames)
$recipe_name: recipename,
$bag_names: JSON.stringify(bagnames)
});
};
@ -217,8 +259,8 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
$title
)
`,{
title: tiddlerFields.title,
bag_name: bagname
$title: tiddlerFields.title,
$bag_name: bagname
});
// Update the fields table
this.runStatement(`
@ -238,9 +280,9 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
) AS t
JOIN json_each($field_values) AS json_each
`,{
title: tiddlerFields.title,
bag_name: bagname,
field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined}))
$title: tiddlerFields.title,
$bag_name: bagname,
$field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined}))
});
return {
tiddler_id: info.lastInsertRowid
@ -268,7 +310,7 @@ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipena
) AS selected_bag
ON b.bag_id = selected_bag.bag_id
`,{
recipe_name: recipename
$recipe_name: recipename
});
if(!row) {
return null;
@ -292,8 +334,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
WHERE b.bag_name = $bag_name AND t.title = $title
)
`,{
title: title,
bag_name: bagname
$title: title,
$bag_name: bagname
});
this.runStatement(`
DELETE FROM tiddlers
@ -303,8 +345,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
WHERE bag_name = $bag_name
) AND title = $title
`,{
title: title,
bag_name: bagname
$title: title,
$bag_name: bagname
});
};
@ -322,8 +364,8 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) {
WHERE t.title = $title AND b.bag_name = $bag_name
)
`,{
title: title,
bag_name: bagname
$title: title,
$bag_name: bagname
});
if(rows.length === 0) {
return null;
@ -353,8 +395,8 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
ORDER BY rb.position DESC
LIMIT 1
`,{
title: title,
recipe_name: recipename
$title: title,
$recipe_name: recipename
});
if(!rowTiddlerId) {
return null;
@ -365,8 +407,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
FROM fields
WHERE tiddler_id = $tiddler_id
`,{
tiddler_id: rowTiddlerId.tiddler_id,
recipe_name: recipename
$tiddler_id: rowTiddlerId.tiddler_id
});
return {
bag_name: rowTiddlerId.bag_name,
@ -392,7 +433,7 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) {
)
ORDER BY title ASC
`,{
bag_name: bagname
$bag_name: bagname
});
return rows.map(value => value.title);
};
@ -414,7 +455,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) {
ORDER BY t.title
)
`,{
recipe_name: recipename
$recipe_name: recipename
});
return rows;
};
@ -428,7 +469,7 @@ SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bagname) {
WHERE bag_name = $bag_name
)
`,{
bag_name: bagname
$bag_name: bagname
});
};
@ -448,7 +489,7 @@ SqlTiddlerDatabase.prototype.getRecipeBags = function(recipename) {
) AS bag_priority ON bags.bag_id = bag_priority.bag_id
ORDER BY position
`,{
recipe_name: recipename
$recipe_name: recipename
});
return rows.map(value => value.bag_name);
};
@ -459,16 +500,21 @@ Execute the given function in a transaction, committing if successful but rollin
Calls to this function can be safely nested, but only the top-most call will actually take place in a transaction.
*/
SqlTiddlerDatabase.prototype.transaction = function(fn) {
try {
const alreadyInTransaction = this.transactionDepth > 0;
this.transactionDepth++;
if(alreadyInTransaction) {
return fn();
} else {
return this.db.transaction(fn)();
const alreadyInTransaction = this.transactionDepth > 0;
this.transactionDepth++;
if(alreadyInTransaction) {
return fn();
} else {
try {
this.runStatement(`BEGIN TRANSACTION`);
var result = fn();
this.runStatement(`COMMIT TRANSACTION`);
} catch(e) {
this.runStatement(`ROLLBACK TRANSACTION`);
} finally {
this.transactionDepth--;
}
} finally {
this.transactionDepth--;
return result;
}
};

4
plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-store.js
View File

@ -20,6 +20,7 @@ Create a tiddler store. Options include:
databasePath - path to the database file (can be ":memory:" to get a temporary database)
adminWiki - reference to $tw.Wiki object into which entity state tiddlers should be saved
engine - wasm | better
*/
function SqlTiddlerStore(options) {
options = options || {};
@ -29,7 +30,8 @@ function SqlTiddlerStore(options) {
this.databasePath = options.databasePath || ":memory:";
var SqlTiddlerDatabase = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-database.js").SqlTiddlerDatabase;
this.sqlTiddlerDatabase = new SqlTiddlerDatabase({
databasePath: this.databasePath
databasePath: this.databasePath,
engine: options.engine
});
this.sqlTiddlerDatabase.createTables();
this.updateAdminWiki();