MWS: Add support for node-sqlite-wasm alongside better-sqlite3 (#7996)
* Switch from better-sqlite3 to node-sqlite3-wasm Seems to be slower, but might make cloud deployments easier by not having any binary dependencies * More logging * Temporarily use a memory database We will make this configurable * Revert "More logging" * Resume loading demo tiddlers * Cache prepared statements Gives a 20% reduction in startup time on my machine * Some more logging * Update package-lock * More logging * Route regexps should allow for proxies that automatically decode URLs Astonishingly, Azure does this * Go back to a file-based database * Less logging * Update package-lock.json * Simplify startup by not loading the docs edition * Tiddler database layer should mark statements as having been removed * Re-introduce better-sqlite3 * Make the SQLite provider be switchable * Support switchable SQL engines I am not intending to make this a long term feature. We will choose one engine and stick with it until we choose to change to another. * Adjust dependency versions * Setting up default engine * Make transaction handling compatible with node-sqlite3-wasm https://github.com/tndrle/node-sqlite3-wasm doesn't have transaction support so I've tried to implement it using SQL statements directly. @hoelzro do you think this is right? Should we be rolling back the transaction in the finally clause? It would be nice to have tests in this area... I looked at better-sqlite3's implementation - https://github.com/WiseLibs/better-sqlite3/blob/master/lib/methods/transaction.js * Default to better-sqlite3 for compatibility after merging
This commit is contained in:
Jeremy Ruston
GitHub
parent
790f431df0
commit
3fca82321e
|
|
@ -9,7 +9,8 @@
|
|||
"version": "5.3.4-prerelease",
|
||||
"license": "BSD",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^9.4.3"
|
||||
"better-sqlite3": "^9.4.3",
|
||||
"node-sqlite3-wasm": "^0.8.10"
|
||||
},
|
||||
"bin": {
|
||||
"tiddlywiki": "tiddlywiki.js"
|
||||
|
|
@ -1114,6 +1115,11 @@
|
|||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/node-sqlite3-wasm": {
|
||||
"version": "0.8.10",
|
||||
"resolved": "https://registry.npmjs.org/node-sqlite3-wasm/-/node-sqlite3-wasm-0.8.10.tgz",
|
||||
"integrity": "sha512-is4xaYRCIHxYsL9rY7AbutnPIoOqBqEzIiQrm/X5QzFQEgyF18Rob3Wj096NuShTWq6ZmQIMcvTaF5vfHbSdjw=="
|
||||
},
|
||||
"node_modules/once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
"lint": "eslint ."
|
||||
},
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^9.4.3"
|
||||
"better-sqlite3": "^9.4.3",
|
||||
"node-sqlite3-wasm": "^0.8.10"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,21 +20,6 @@ exports.synchronous = true;
|
|||
|
||||
exports.startup = function() {
|
||||
var path = require("path");
|
||||
// Install the sqlite3 global namespace
|
||||
$tw.sqlite3 = {
|
||||
Database: null
|
||||
};
|
||||
// Check that better-sqlite3 is installed
|
||||
var logger = new $tw.utils.Logger("multiwikiserver");
|
||||
try {
|
||||
$tw.sqlite3.Database = require("better-sqlite3");
|
||||
} catch(e) {
|
||||
}
|
||||
console.log(`Successfully required better-sqlite3`)
|
||||
if(!$tw.sqlite3.Database) {
|
||||
logger.alert("The plugin 'tiddlywiki/multiwikiserver' requires the better-sqlite3 npm package to be installed. Run 'npm install' in the root of the TiddlyWiki repository");
|
||||
return;
|
||||
}
|
||||
// Create and initialise the tiddler store and upload manager
|
||||
var SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore,
|
||||
store = new SqlTiddlerStore({
|
||||
|
|
@ -51,13 +36,27 @@ console.log(`Successfully required better-sqlite3`)
|
|||
};
|
||||
// Performance timing
|
||||
console.time("mws-initial-load");
|
||||
// Create docs bag and recipe
|
||||
$tw.mws.store.createBag("docs","TiddlyWiki Documentation from https://tiddlywiki.com/");
|
||||
$tw.mws.store.createRecipe("docs",["docs"],"TiddlyWiki Documentation from https://tiddlywiki.com/");
|
||||
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"tw5.com/tiddlers"),"docs");
|
||||
$tw.mws.store.createBag("dev-docs","TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/");
|
||||
$tw.mws.store.createRecipe("dev-docs",["dev-docs"],"TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev/");
|
||||
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,"dev/tiddlers"),"dev-docs");
|
||||
// Copy TiddlyWiki core editions
|
||||
function copyEdition(options) {
|
||||
console.log(`Copying edition ${options.tiddlersPath}`);
|
||||
$tw.mws.store.createBag(options.bagName,options.bagDescription);
|
||||
$tw.mws.store.createRecipe(options.recipeName,[options.bagName],options.recipeDescription);
|
||||
$tw.mws.store.saveTiddlersFromPath(path.resolve($tw.boot.corePath,$tw.config.editionsPath,options.tiddlersPath),options.bagName);
|
||||
}
|
||||
// copyEdition({
|
||||
// bagName: "docs",
|
||||
// bagDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
|
||||
// recipeName: "docs",
|
||||
// recipeDescription: "TiddlyWiki Documentation from https://tiddlywiki.com",
|
||||
// tiddlersPath: "tw5.com/tiddlers"
|
||||
// });
|
||||
copyEdition({
|
||||
bagName: "dev-docs",
|
||||
bagDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
|
||||
recipeName: "dev-docs",
|
||||
recipeDescription: "TiddlyWiki Developer Documentation from https://tiddlywiki.com/dev",
|
||||
tiddlersPath: "dev/tiddlers"
|
||||
});
|
||||
// Create bags and recipes
|
||||
$tw.mws.store.createBag("bag-alpha","A test bag");
|
||||
$tw.mws.store.createBag("bag-beta","Another test bag");
|
||||
|
|
|
|||
|
|
@ -16,38 +16,80 @@ Validation is for the most part left to the caller
|
|||
Create a tiddler store. Options include:
|
||||
|
||||
databasePath - path to the database file (can be ":memory:" to get a temporary database)
|
||||
engine - wasm | better
|
||||
*/
|
||||
function SqlTiddlerDatabase(options) {
|
||||
options = options || {};
|
||||
// Create the database
|
||||
// Initialise the statement cache
|
||||
this.statements = Object.create(null); // Hashmap by SQL text of statement objects
|
||||
// Create the database file directories if needed
|
||||
if(options.databasePath) {
|
||||
$tw.utils.createFileDirectories(options.databasePath);
|
||||
}
|
||||
var databasePath = options.databasePath || ":memory:";
|
||||
this.db = new $tw.sqlite3.Database(databasePath,{verbose: undefined && console.log});
|
||||
// Choose engine
|
||||
this.engine = options.engine || "better"; // wasm | better
|
||||
// Create the database
|
||||
const databasePath = options.databasePath || ":memory:";
|
||||
let Database;
|
||||
console.log(`Creating SQL engine ${this.engine}`)
|
||||
switch(this.engine) {
|
||||
case "wasm":
|
||||
({ Database } = require("node-sqlite3-wasm"));
|
||||
break;
|
||||
case "better":
|
||||
Database = require("better-sqlite3");
|
||||
break;
|
||||
}
|
||||
this.db = new Database(databasePath,{
|
||||
verbose: undefined && console.log
|
||||
});
|
||||
this.transactionDepth = 0;
|
||||
}
|
||||
|
||||
SqlTiddlerDatabase.prototype.close = function() {
|
||||
for(const sql in this.statements) {
|
||||
this.statements[sql].finalize();
|
||||
}
|
||||
this.statements = Object.create(null);
|
||||
this.db.close();
|
||||
this.db = undefined;
|
||||
};
|
||||
|
||||
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) {
|
||||
SqlTiddlerDatabase.prototype.normaliseParams = function(params) {
|
||||
params = params || {};
|
||||
const statement = this.db.prepare(sql);
|
||||
const result = Object.create(null);
|
||||
for(const paramName in params) {
|
||||
if(this.engine !== "wasm" && paramName.startsWith("$")) {
|
||||
result[paramName.slice(1)] = params[paramName];
|
||||
} else {
|
||||
result[paramName] = params[paramName];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
SqlTiddlerDatabase.prototype.prepareStatement = function(sql) {
|
||||
if(!(sql in this.statements)) {
|
||||
this.statements[sql] = this.db.prepare(sql);
|
||||
}
|
||||
return this.statements[sql];
|
||||
};
|
||||
|
||||
SqlTiddlerDatabase.prototype.runStatement = function(sql,params) {
|
||||
params = this.normaliseParams(params);
|
||||
const statement = this.prepareStatement(sql);
|
||||
return statement.run(params);
|
||||
};
|
||||
|
||||
SqlTiddlerDatabase.prototype.runStatementGet = function(sql,params) {
|
||||
params = params || {};
|
||||
const statement = this.db.prepare(sql);
|
||||
params = this.normaliseParams(params);
|
||||
const statement = this.prepareStatement(sql);
|
||||
return statement.get(params);
|
||||
};
|
||||
|
||||
SqlTiddlerDatabase.prototype.runStatementGetAll = function(sql,params) {
|
||||
params = params || {};
|
||||
const statement = this.db.prepare(sql);
|
||||
params = this.normaliseParams(params);
|
||||
const statement = this.prepareStatement(sql);
|
||||
return statement.all(params);
|
||||
};
|
||||
|
||||
|
|
@ -134,7 +176,7 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
|
|||
INSERT OR IGNORE INTO bags (bag_name, accesscontrol, description)
|
||||
VALUES ($bag_name, '', '')
|
||||
`,{
|
||||
bag_name: bagname
|
||||
$bag_name: bagname
|
||||
});
|
||||
this.runStatement(`
|
||||
UPDATE bags
|
||||
|
|
@ -142,9 +184,9 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description) {
|
|||
description = $description
|
||||
WHERE bag_name = $bag_name
|
||||
`,{
|
||||
bag_name: bagname,
|
||||
accesscontrol: "[some access control stuff]",
|
||||
description: description
|
||||
$bag_name: bagname,
|
||||
$accesscontrol: "[some access control stuff]",
|
||||
$description: description
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -182,15 +224,15 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
|
|||
-- Delete existing recipe_bags entries for this recipe
|
||||
DELETE FROM recipe_bags WHERE recipe_id = (SELECT recipe_id FROM recipes WHERE recipe_name = $recipe_name)
|
||||
`,{
|
||||
recipe_name: recipename
|
||||
$recipe_name: recipename
|
||||
});
|
||||
this.runStatement(`
|
||||
-- Create the entry in the recipes table if required
|
||||
INSERT OR REPLACE INTO recipes (recipe_name, description)
|
||||
VALUES ($recipe_name, $description)
|
||||
`,{
|
||||
recipe_name: recipename,
|
||||
description: description
|
||||
$recipe_name: recipename,
|
||||
$description: description
|
||||
});
|
||||
this.runStatement(`
|
||||
INSERT INTO recipe_bags (recipe_id, bag_id, position)
|
||||
|
|
@ -200,8 +242,8 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
|
|||
INNER JOIN json_each($bag_names) AS j ON j.value = b.bag_name
|
||||
WHERE r.recipe_name = $recipe_name
|
||||
`,{
|
||||
recipe_name: recipename,
|
||||
bag_names: JSON.stringify(bagnames)
|
||||
$recipe_name: recipename,
|
||||
$bag_names: JSON.stringify(bagnames)
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -217,8 +259,8 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
|
|||
$title
|
||||
)
|
||||
`,{
|
||||
title: tiddlerFields.title,
|
||||
bag_name: bagname
|
||||
$title: tiddlerFields.title,
|
||||
$bag_name: bagname
|
||||
});
|
||||
// Update the fields table
|
||||
this.runStatement(`
|
||||
|
|
@ -238,9 +280,9 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
|
|||
) AS t
|
||||
JOIN json_each($field_values) AS json_each
|
||||
`,{
|
||||
title: tiddlerFields.title,
|
||||
bag_name: bagname,
|
||||
field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined}))
|
||||
$title: tiddlerFields.title,
|
||||
$bag_name: bagname,
|
||||
$field_values: JSON.stringify(Object.assign({},tiddlerFields,{title: undefined}))
|
||||
});
|
||||
return {
|
||||
tiddler_id: info.lastInsertRowid
|
||||
|
|
@ -268,7 +310,7 @@ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipena
|
|||
) AS selected_bag
|
||||
ON b.bag_id = selected_bag.bag_id
|
||||
`,{
|
||||
recipe_name: recipename
|
||||
$recipe_name: recipename
|
||||
});
|
||||
if(!row) {
|
||||
return null;
|
||||
|
|
@ -292,8 +334,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
|
|||
WHERE b.bag_name = $bag_name AND t.title = $title
|
||||
)
|
||||
`,{
|
||||
title: title,
|
||||
bag_name: bagname
|
||||
$title: title,
|
||||
$bag_name: bagname
|
||||
});
|
||||
this.runStatement(`
|
||||
DELETE FROM tiddlers
|
||||
|
|
@ -303,8 +345,8 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
|
|||
WHERE bag_name = $bag_name
|
||||
) AND title = $title
|
||||
`,{
|
||||
title: title,
|
||||
bag_name: bagname
|
||||
$title: title,
|
||||
$bag_name: bagname
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -322,8 +364,8 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) {
|
|||
WHERE t.title = $title AND b.bag_name = $bag_name
|
||||
)
|
||||
`,{
|
||||
title: title,
|
||||
bag_name: bagname
|
||||
$title: title,
|
||||
$bag_name: bagname
|
||||
});
|
||||
if(rows.length === 0) {
|
||||
return null;
|
||||
|
|
@ -353,8 +395,8 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
|
|||
ORDER BY rb.position DESC
|
||||
LIMIT 1
|
||||
`,{
|
||||
title: title,
|
||||
recipe_name: recipename
|
||||
$title: title,
|
||||
$recipe_name: recipename
|
||||
});
|
||||
if(!rowTiddlerId) {
|
||||
return null;
|
||||
|
|
@ -365,8 +407,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
|
|||
FROM fields
|
||||
WHERE tiddler_id = $tiddler_id
|
||||
`,{
|
||||
tiddler_id: rowTiddlerId.tiddler_id,
|
||||
recipe_name: recipename
|
||||
$tiddler_id: rowTiddlerId.tiddler_id
|
||||
});
|
||||
return {
|
||||
bag_name: rowTiddlerId.bag_name,
|
||||
|
|
@ -392,7 +433,7 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) {
|
|||
)
|
||||
ORDER BY title ASC
|
||||
`,{
|
||||
bag_name: bagname
|
||||
$bag_name: bagname
|
||||
});
|
||||
return rows.map(value => value.title);
|
||||
};
|
||||
|
|
@ -414,7 +455,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) {
|
|||
ORDER BY t.title
|
||||
)
|
||||
`,{
|
||||
recipe_name: recipename
|
||||
$recipe_name: recipename
|
||||
});
|
||||
return rows;
|
||||
};
|
||||
|
|
@ -428,7 +469,7 @@ SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bagname) {
|
|||
WHERE bag_name = $bag_name
|
||||
)
|
||||
`,{
|
||||
bag_name: bagname
|
||||
$bag_name: bagname
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -448,7 +489,7 @@ SqlTiddlerDatabase.prototype.getRecipeBags = function(recipename) {
|
|||
) AS bag_priority ON bags.bag_id = bag_priority.bag_id
|
||||
ORDER BY position
|
||||
`,{
|
||||
recipe_name: recipename
|
||||
$recipe_name: recipename
|
||||
});
|
||||
return rows.map(value => value.bag_name);
|
||||
};
|
||||
|
|
@ -459,16 +500,21 @@ Execute the given function in a transaction, committing if successful but rollin
|
|||
Calls to this function can be safely nested, but only the top-most call will actually take place in a transaction.
|
||||
*/
|
||||
SqlTiddlerDatabase.prototype.transaction = function(fn) {
|
||||
try {
|
||||
const alreadyInTransaction = this.transactionDepth > 0;
|
||||
this.transactionDepth++;
|
||||
if(alreadyInTransaction) {
|
||||
return fn();
|
||||
} else {
|
||||
return this.db.transaction(fn)();
|
||||
const alreadyInTransaction = this.transactionDepth > 0;
|
||||
this.transactionDepth++;
|
||||
if(alreadyInTransaction) {
|
||||
return fn();
|
||||
} else {
|
||||
try {
|
||||
this.runStatement(`BEGIN TRANSACTION`);
|
||||
var result = fn();
|
||||
this.runStatement(`COMMIT TRANSACTION`);
|
||||
} catch(e) {
|
||||
this.runStatement(`ROLLBACK TRANSACTION`);
|
||||
} finally {
|
||||
this.transactionDepth--;
|
||||
}
|
||||
} finally {
|
||||
this.transactionDepth--;
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ Create a tiddler store. Options include:
|
|||
|
||||
databasePath - path to the database file (can be ":memory:" to get a temporary database)
|
||||
adminWiki - reference to $tw.Wiki object into which entity state tiddlers should be saved
|
||||
engine - wasm | better
|
||||
*/
|
||||
function SqlTiddlerStore(options) {
|
||||
options = options || {};
|
||||
|
|
@ -29,7 +30,8 @@ function SqlTiddlerStore(options) {
|
|||
this.databasePath = options.databasePath || ":memory:";
|
||||
var SqlTiddlerDatabase = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-database.js").SqlTiddlerDatabase;
|
||||
this.sqlTiddlerDatabase = new SqlTiddlerDatabase({
|
||||
databasePath: this.databasePath
|
||||
databasePath: this.databasePath,
|
||||
engine: options.engine
|
||||
});
|
||||
this.sqlTiddlerDatabase.createTables();
|
||||
this.updateAdminWiki();
|
||||
|
|
|
|||
Loading…
Reference in New Issue