From 69cc45bf5c45f21bd3d1bbd145833831b3a73f7b Mon Sep 17 00:00:00 2001 From: Jeremy Ruston Date: Sun, 17 Mar 2024 13:27:00 +0000 Subject: [PATCH] Refactor the database engine specific code --- .../modules/store/sql-engine.js | 134 ++++++++++++++ .../modules/store/sql-tiddler-database.js | 167 +++--------------- .../modules/store/sql-tiddler-store.js | 4 - 3 files changed, 163 insertions(+), 142 deletions(-) create mode 100644 plugins/tiddlywiki/multiwikiserver/modules/store/sql-engine.js diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-engine.js b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-engine.js new file mode 100644 index 000000000..2390edb16 --- /dev/null +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-engine.js @@ -0,0 +1,134 @@ +/*\ +title: $:/plugins/tiddlywiki/multiwikiserver/store/sql-engine.js +type: application/javascript +module-type: library + +Low level functions to work with the SQLite engine, either via better-sqlite3 or node-sqlite3-wasm. + +This class is intended to encapsulate all engine-specific logic. + +\*/ + +(function() { + +/* +Create a database engine. Options include: + +databasePath - path to the database file (can be ":memory:" or missing to get a temporary database) +engine - wasm | better +*/ +function SqlEngine(options) { + options = options || {}; + // Initialise transaction mechanism + this.transactionDepth = 0; + // Initialise the statement cache + this.statements = Object.create(null); // Hashmap by SQL text of statement objects + // Choose engine + this.engine = options.engine || "better"; // wasm | better + // Create the database file directories if needed + if(options.databasePath) { + $tw.utils.createFileDirectories(options.databasePath); + } + // Create the database + const databasePath = options.databasePath || ":memory:"; + let Database; + switch(this.engine) { + case "wasm": + ({ Database } = require("node-sqlite3-wasm")); + break; + case "better": + Database = require("better-sqlite3"); + break; + } + this.db = new Database(databasePath,{ + verbose: undefined && console.log + }); +} + +SqlEngine.prototype.close = function() { + for(const sql in this.statements) { + if(this.statements[sql].finalize) { + this.statements[sql].finalize(); + } + } + this.statements = Object.create(null); + this.db.close(); + this.db = undefined; +}; + +SqlEngine.prototype.normaliseParams = function(params) { + params = params || {}; + const result = Object.create(null); + for(const paramName in params) { + if(this.engine !== "wasm" && paramName.startsWith("$")) { + result[paramName.slice(1)] = params[paramName]; + } else { + result[paramName] = params[paramName]; + } + } + return result; +}; + +SqlEngine.prototype.prepareStatement = function(sql) { + if(!(sql in this.statements)) { + this.statements[sql] = this.db.prepare(sql); + } + return this.statements[sql]; +}; + +SqlEngine.prototype.runStatement = function(sql,params) { + params = this.normaliseParams(params); + const statement = this.prepareStatement(sql); + return statement.run(params); +}; + +SqlEngine.prototype.runStatementGet = function(sql,params) { + params = this.normaliseParams(params); + const statement = this.prepareStatement(sql); + return statement.get(params); +}; + +SqlEngine.prototype.runStatementGetAll = function(sql,params) { + params = this.normaliseParams(params); + const statement = this.prepareStatement(sql); + return statement.all(params); +}; + +SqlEngine.prototype.runStatements = function(sqlArray) { + for(const sql of sqlArray) { + this.runStatement(sql); + } +}; + +/* +Execute the given function in a transaction, committing if successful but rolling back if an error occurs. Returns whatever the given function returns. + +Calls to this function can be safely nested, but only the topmost call will actually take place in a transaction. + +TODO: better-sqlite3 provides its own transaction method which we should be using if available +*/ +SqlEngine.prototype.transaction = function(fn) { + const alreadyInTransaction = this.transactionDepth > 0; + this.transactionDepth++; + try { + if(alreadyInTransaction) { + return fn(); + } else { + this.runStatement(`BEGIN TRANSACTION`); + try { + var result = fn(); + this.runStatement(`COMMIT TRANSACTION`); + } catch(e) { + this.runStatement(`ROLLBACK TRANSACTION`); + throw(e); + } + return result; + } + } finally { + this.transactionDepth--; + } +}; + +exports.SqlEngine = SqlEngine; + +})(); \ No newline at end of file diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js index 145985550..d1271e59d 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js @@ -20,89 +20,24 @@ engine - wasm | better */ function SqlTiddlerDatabase(options) { options = options || {}; - // Initialise the statement cache - this.statements = Object.create(null); // Hashmap by SQL text of statement objects - // Create the database file directories if needed - if(options.databasePath) { - $tw.utils.createFileDirectories(options.databasePath); - } - // Choose engine - this.engine = options.engine || "better"; // wasm | better - // Create the database - const databasePath = options.databasePath || ":memory:"; - let Database; - console.log(`Creating SQL engine ${this.engine}`) - switch(this.engine) { - case "wasm": - ({ Database } = require("node-sqlite3-wasm")); - break; - case "better": - Database = require("better-sqlite3"); - break; - } - this.db = new Database(databasePath,{ - verbose: undefined && console.log + const SqlEngine = require("$:/plugins/tiddlywiki/multiwikiserver/store/sql-engine.js").SqlEngine; + this.engine = new SqlEngine({ + databasePath: options.databasePath, + engine: options.engine }); - this.transactionDepth = 0; } SqlTiddlerDatabase.prototype.close = function() { - for(const sql in this.statements) { - if(this.statements[sql].finalize) { - this.statements[sql].finalize(); - } - } - this.statements = Object.create(null); - this.db.close(); - this.db = undefined; + this.engine.close(); }; -SqlTiddlerDatabase.prototype.normaliseParams = function(params) { - params = params || {}; - const result = Object.create(null); - for(const paramName in params) { - if(this.engine !== "wasm" && paramName.startsWith("$")) { - result[paramName.slice(1)] = params[paramName]; - } else { - result[paramName] = params[paramName]; - } - } - return result; -}; -SqlTiddlerDatabase.prototype.prepareStatement = function(sql) { - if(!(sql in this.statements)) { - this.statements[sql] = this.db.prepare(sql); - } - return this.statements[sql]; -}; - -SqlTiddlerDatabase.prototype.runStatement = function(sql,params) { - params = this.normaliseParams(params); - const statement = this.prepareStatement(sql); - return statement.run(params); -}; - -SqlTiddlerDatabase.prototype.runStatementGet = function(sql,params) { - params = this.normaliseParams(params); - const statement = this.prepareStatement(sql); - return statement.get(params); -}; - -SqlTiddlerDatabase.prototype.runStatementGetAll = function(sql,params) { - params = this.normaliseParams(params); - const statement = this.prepareStatement(sql); - return statement.all(params); -}; - -SqlTiddlerDatabase.prototype.runStatements = function(sqlArray) { - for(const sql of sqlArray) { - this.runStatement(sql); - } +SqlTiddlerDatabase.prototype.transaction = function(fn) { + return this.engine.transaction(fn); }; SqlTiddlerDatabase.prototype.createTables = function() { - this.runStatements([` + this.engine.runStatements([` -- Bags have names and access control settings CREATE TABLE IF NOT EXISTS bags ( bag_id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -149,23 +84,8 @@ SqlTiddlerDatabase.prototype.createTables = function() { `]); }; -SqlTiddlerDatabase.prototype.logTables = function() { - var self = this; - function sqlLogTable(table) { - console.log(`TABLE ${table}:`); - let statement = self.db.prepare(`select * from ${table}`); - for(const row of statement.all()) { - console.log(row); - } - } - const tables = ["recipes","bags","recipe_bags","tiddlers","fields"]; - for(const table of tables) { - sqlLogTable(table); - } -}; - SqlTiddlerDatabase.prototype.listBags = function() { - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT bag_name, accesscontrol, description FROM bags ORDER BY bag_name @@ -176,13 +96,13 @@ SqlTiddlerDatabase.prototype.listBags = function() { SqlTiddlerDatabase.prototype.createBag = function(bagname,description,accesscontrol) { accesscontrol = accesscontrol || ""; // Run the queries - this.runStatement(` + this.engine.runStatement(` INSERT OR IGNORE INTO bags (bag_name, accesscontrol, description) VALUES ($bag_name, '', '') `,{ $bag_name: bagname }); - this.runStatement(` + this.engine.runStatement(` UPDATE bags SET accesscontrol = $accesscontrol, description = $description @@ -198,7 +118,7 @@ SqlTiddlerDatabase.prototype.createBag = function(bagname,description,accesscont Returns array of {recipe_name:,description:,bag_names: []} */ SqlTiddlerDatabase.prototype.listRecipes = function() { - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT r.recipe_name, r.description, b.bag_name, rb.position FROM recipes AS r JOIN recipe_bags AS rb ON rb.recipe_id = r.recipe_id @@ -224,13 +144,13 @@ SqlTiddlerDatabase.prototype.listRecipes = function() { SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,description) { // Run the queries - this.runStatement(` + this.engine.runStatement(` -- Delete existing recipe_bags entries for this recipe DELETE FROM recipe_bags WHERE recipe_id = (SELECT recipe_id FROM recipes WHERE recipe_name = $recipe_name) `,{ $recipe_name: recipename }); - this.runStatement(` + this.engine.runStatement(` -- Create the entry in the recipes table if required INSERT OR REPLACE INTO recipes (recipe_name, description) VALUES ($recipe_name, $description) @@ -238,7 +158,7 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip $recipe_name: recipename, $description: description }); - this.runStatement(` + this.engine.runStatement(` INSERT INTO recipe_bags (recipe_id, bag_id, position) SELECT r.recipe_id, b.bag_id, j.key as position FROM recipes r @@ -257,7 +177,7 @@ Returns {tiddler_id:} SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname,attachment_blob) { attachment_blob = attachment_blob || null; // Update the tiddlers table - var info = this.runStatement(` + var info = this.engine.runStatement(` INSERT OR REPLACE INTO tiddlers (bag_id, title, attachment_blob) VALUES ( (SELECT bag_id FROM bags WHERE bag_name = $bag_name), @@ -270,7 +190,7 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname,att $bag_name: bagname }); // Update the fields table - this.runStatement(` + this.engine.runStatement(` INSERT OR REPLACE INTO fields (tiddler_id, field_name, field_value) SELECT t.tiddler_id, @@ -301,7 +221,7 @@ Returns {tiddler_id:,bag_name:} or null if the recipe is empty */ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipename,attachment_blob) { // Find the topmost bag in the recipe - var row = this.runStatementGet(` + var row = this.engine.runStatementGet(` SELECT b.bag_name FROM bags AS b JOIN ( @@ -332,7 +252,7 @@ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipena SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) { // Run the queries - this.runStatement(` + this.engine.runStatement(` DELETE FROM fields WHERE tiddler_id IN ( SELECT t.tiddler_id @@ -344,7 +264,7 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) { $title: title, $bag_name: bagname }); - this.runStatement(` + this.engine.runStatement(` DELETE FROM tiddlers WHERE bag_id = ( SELECT bag_id @@ -361,7 +281,7 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) { returns {tiddler_id:,tiddler:,attachment_blob:} */ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) { - const rowTiddler = this.runStatementGet(` + const rowTiddler = this.engine.runStatementGet(` SELECT t.tiddler_id, t.attachment_blob FROM bags AS b INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id @@ -373,7 +293,7 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) { if(!rowTiddler) { return null; } - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT field_name, field_value, tiddler_id FROM fields WHERE tiddler_id = $tiddler_id @@ -398,7 +318,7 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) { Returns {bag_name:, tiddler: {fields}, tiddler_id:, attachment_blob:} */ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) { - const rowTiddlerId = this.runStatementGet(` + const rowTiddlerId = this.engine.runStatementGet(` SELECT t.tiddler_id, t.attachment_blob, b.bag_name FROM bags AS b INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id @@ -416,7 +336,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) { return null; } // Get the fields - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT field_name, field_value FROM fields WHERE tiddler_id = $tiddler_id @@ -438,7 +358,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) { Get the titles of the tiddlers in a bag. Returns an empty array for bags that do not exist */ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) { - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT DISTINCT title FROM tiddlers WHERE bag_id IN ( @@ -457,7 +377,7 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) { Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist */ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) { - const rowsCheckRecipe = this.runStatementGetAll(` + const rowsCheckRecipe = this.engine.runStatementGetAll(` SELECT * FROM recipes WHERE recipes.recipe_name = $recipe_name `,{ $recipe_name: recipename @@ -465,7 +385,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) { if(rowsCheckRecipe.length === 0) { return null; } - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT title, bag_name FROM ( SELECT t.title, b.bag_name, MAX(rb.position) AS position @@ -484,7 +404,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) { }; SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bagname) { - this.runStatement(` + this.engine.runStatement(` DELETE FROM tiddlers WHERE bag_id IN ( SELECT bag_id @@ -500,7 +420,7 @@ SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bagname) { Get the names of the bags in a recipe. Returns an empty array for recipes that do not exist */ SqlTiddlerDatabase.prototype.getRecipeBags = function(recipename) { - const rows = this.runStatementGetAll(` + const rows = this.engine.runStatementGetAll(` SELECT bags.bag_name FROM bags JOIN ( @@ -517,35 +437,6 @@ SqlTiddlerDatabase.prototype.getRecipeBags = function(recipename) { return rows.map(value => value.bag_name); }; -/* -Execute the given function in a transaction, committing if successful but rolling back if an error occurs. Returns whatever the given function returns. - -Calls to this function can be safely nested, but only the topmost call will actually take place in a transaction. - -TODO: better-sqlite3 provides its own transaction method which we should be using if available -*/ -SqlTiddlerDatabase.prototype.transaction = function(fn) { - const alreadyInTransaction = this.transactionDepth > 0; - this.transactionDepth++; - try { - if(alreadyInTransaction) { - return fn(); - } else { - this.runStatement(`BEGIN TRANSACTION`); - try { - var result = fn(); - this.runStatement(`COMMIT TRANSACTION`); - } catch(e) { - this.runStatement(`ROLLBACK TRANSACTION`); - throw(e); - } - return result; - } - } finally { - this.transactionDepth--; - } -}; - exports.SqlTiddlerDatabase = SqlTiddlerDatabase; })(); \ No newline at end of file diff --git a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js index c7029baa6..6731f0328 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js @@ -170,10 +170,6 @@ SqlTiddlerStore.prototype.saveTiddlersFromPath = function(tiddler_files_path,bag }); }; -SqlTiddlerStore.prototype.logTables = function() { - this.sqlTiddlerDatabase.logTables(); -}; - SqlTiddlerStore.prototype.listBags = function() { return this.sqlTiddlerDatabase.listBags(); };