mirrors
/
TiddlyWiki5
mirror of https://github.com/Jermolene/TiddlyWiki5
1
0
Fork 0
Code Issues Releases Wiki Activity

More efficient syncing 

Thank you @PotOfCoffee2Go I ended up taking some of your code from #8101 to get this up and running. There's still some stuff missing (like the tests!) but it gets things moving.
This commit is contained in:
Jeremy Ruston 2024-03-23 09:27:54 +00:00
parent 52f76380c7
commit 08649dd1eb
7 changed files with 156 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
plugins/tiddlywiki/multiwikiclient/multiwikiclientadaptor.js
View File

@ -14,13 +14,14 @@ A sync adaptor module for synchronising with MultiWikiServer-compatible servers
var CONFIG_HOST_TIDDLER = "$:/config/multiwikiclient/host",
DEFAULT_HOST_TIDDLER = "$protocol$//$host$/",
BAG_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/bag",
REVISION_STATE_TIDDLER = "$:/state/federatial/xememex/tiddlers/revision";
BAG_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/bag",
REVISION_STATE_TIDDLER = "$:/state/multiwikiclient/tiddlers/revision";
function MultiWikiClientAdaptor(options) {
this.wiki = options.wiki;
this.host = this.getHost();
this.recipe = this.wiki.getTiddlerText("$:/config/multiwikiclient/recipe");
this.last_known_tiddler_id = $tw.utils.parseNumber(this.wiki.getTiddlerText("$:/state/multiwikiclient/recipe/last_tiddler_id","0"));
this.logger = new $tw.utils.Logger("MultiWikiClientAdaptor");
this.isLoggedIn = false;
this.isReadOnly = false;
@ -68,6 +69,10 @@ MultiWikiClientAdaptor.prototype.getTiddlerInfo = function(tiddler) {
}
};
MultiWikiClientAdaptor.prototype.getTiddlerBag = function(title) {
return this.wiki.extractTiddlerDataItem(BAG_STATE_TIDDLER,title);
};
MultiWikiClientAdaptor.prototype.getTiddlerRevision = function(title) {
return this.wiki.extractTiddlerDataItem(REVISION_STATE_TIDDLER,title);
};
@ -163,23 +168,38 @@ MultiWikiClientAdaptor.prototype.getCsrfToken = function() {
};
/*
Get an array of skinny tiddler fields from the server
Get details of changed tiddlers from the server
*/
MultiWikiClientAdaptor.prototype.getSkinnyTiddlers = function(callback) {
MultiWikiClientAdaptor.prototype.getUpdatedTiddlers = function(syncer,callback) {
var self = this;
$tw.utils.httpRequest({
url: this.host + "recipes/" + this.recipe + "/tiddlers.json",
data: {
filter: "[all[tiddlers]] -[[$:/isEncrypted]] -[prefix[$:/temp/]] -[prefix[$:/status/]] -[[$:/boot/boot.js]] -[[$:/boot/bootprefix.js]] -[[$:/library/sjcl.js]] -[[$:/core]]"
last_known_tiddler_id: this.last_known_tiddler_id
},
callback: function(err,data) {
// Check for errors
if(err) {
return callback(err);
}
var tiddlers = $tw.utils.parseJSONSafe(data);
// Invoke the callback with the skinny tiddlers
callback(null,tiddlers);
var modifications = [],
deletions = [];
var tiddlerInfoArray = $tw.utils.parseJSONSafe(data);
$tw.utils.each(tiddlerInfoArray,function(tiddlerInfo) {
if(tiddlerInfo.tiddler_id > self.last_known_tiddler_id) {
self.last_known_tiddler_id = tiddlerInfo.tiddler_id;
}
if(tiddlerInfo.is_deleted) {
deletions.push(tiddlerInfo.title);
} else {
modifications.push(tiddlerInfo.title);
}
});
// Invoke the callback with the results
callback(null,{
modifications: modifications,
deletions: deletions
});
// If Browswer Storage tiddlers were cached on reloading the wiki, add them after sync from server completes in the above callback.
if($tw.browserStorage && $tw.browserStorage.isEnabled()) {
$tw.browserStorage.addCachedTiddlers();
@ -252,7 +272,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options
return callback(null);
}
// If we don't have a bag it means that the tiddler hasn't been seen by the server, so we don't need to delete it
var bag = options.tiddlerInfo.adaptorInfo && options.tiddlerInfo.adaptorInfo.bag;
var bag = this.getTiddlerBag(title);
if(!bag) {
return callback(null,options.tiddlerInfo.adaptorInfo);
}
@ -264,6 +284,7 @@ MultiWikiClientAdaptor.prototype.deleteTiddler = function(title,callback,options
if(err) {
return callback(err);
}
self.removeTiddlerInfo(title);
// Invoke the callback & return null adaptorInfo
callback(null,null);
}

17
plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-recipe-tiddlers-json.js
View File

@ -20,17 +20,14 @@ exports.handler = function(request,response,state) {
// Get the parameters
var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]);
if(recipe_name) {
// Get the tiddlers in the recipe
var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name);
// Get a skinny version of each tiddler
var tiddlers = [];
$tw.utils.each(recipeTiddlers,function(recipeTiddlerInfo) {
var tiddlerInfo = $tw.mws.store.getRecipeTiddler(recipeTiddlerInfo.title,recipe_name);
tiddlers.push(Object.assign({},tiddlerInfo.tiddler,{text: undefined}));
// Get the tiddlers in the recipe, optionally since the specified last known tiddler_id
var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name,{
last_known_tiddler_id: state.queryParameters.last_known_tiddler_id
});
var text = JSON.stringify(tiddlers);
state.sendResponse(200,{"Content-Type": "application/json"},text,"utf8");
return;
if(recipeTiddlers) {
state.sendResponse(200,{"Content-Type": "application/json"},JSON.stringify(recipeTiddlers),"utf8");
return;
}
}
// Fail if something went wrong
response.writeHead(404);

4
plugins/tiddlywiki/multiwikiserver/modules/routes/handlers/get-wiki.js
View File

@ -77,6 +77,10 @@ exports.handler = function(request,response,state) {
title: "$:/config/multiwikiclient/recipe",
text: recipe_name
});
writeTiddler({
title: "$:/state/multiwikiclient/recipe/last_tiddler_id",
text: $tw.mws.store.getRecipeLastTiddlerId(recipe_name).toString()
});
response.write(template.substring(markerPos + marker.length))
// Finish response
response.end();

107
plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-database.js
View File

@ -395,36 +395,105 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bag_name) {
};
/*
Get the titles of the tiddlers in a recipe as {title:,tiddler_id:,bag_name:}. Returns null for recipes that do not exist
Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist
*/
SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name) {
const rowsCheckRecipe = this.engine.runStatementGetAll(`
SELECT * FROM recipes WHERE recipes.recipe_name = $recipe_name
SqlTiddlerDatabase.prototype.getBagLastTiddlerId = function(bag_name) {
const row = this.engine.runStatementGet(`
SELECT tiddler_id
FROM tiddlers
WHERE bag_id IN (
SELECT bag_id
FROM bags
WHERE bag_name = $bag_name
)
ORDER BY tiddler_id DESC
LIMIT 1
`,{
$recipe_name: recipe_name
$bag_name: bag_name
});
if(rowsCheckRecipe.length === 0) {
if(row) {
return row.tiddler_id;
} else {
return null;
}
const rows = this.engine.runStatementGetAll(`
SELECT title, tiddler_id, bag_name
FROM (
SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position
FROM bags AS b
INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id
INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
WHERE r.recipe_name = $recipe_name
AND t.is_deleted = FALSE
GROUP BY t.title
ORDER BY t.title
)
};
/*
Get the metadata of the tiddlers in a recipe as an array [{title:,tiddler_id:,bag_name:,is_deleted:}],
sorted in ascending order of tiddler_id.
Options include:
limit: optional maximum number of results to return
last_known_tiddler_id: tiddler_id of the last known update. Only returns tiddlers that have been created, modified or deleted since
include_deleted: boolean, defaults to false
Returns null for recipes that do not exist
*/
SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipe_name,options) {
options = options || {};
// Get the recipe ID
const rowsCheckRecipe = this.engine.runStatementGet(`
SELECT recipe_id FROM recipes WHERE recipes.recipe_name = $recipe_name
`,{
$recipe_name: recipe_name
});
if(!rowsCheckRecipe) {
return null;
}
const recipe_id = rowsCheckRecipe.recipe_id;
// Compose the query to get the tiddlers
const params = {
$recipe_id: recipe_id
}
if(options.limit) {
params.$limit = options.limit.toString();
}
if(options.last_known_tiddler_id) {
params.$last_known_tiddler_id = options.last_known_tiddler_id;
}
const rows = this.engine.runStatementGetAll(`
SELECT title, tiddler_id, is_deleted, bag_name
FROM (
SELECT t.title, t.tiddler_id, t.is_deleted, b.bag_name, MAX(rb.position) AS position
FROM bags AS b
INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
WHERE rb.recipe_id = $recipe_id
${options.include_deleted ? "" : "AND t.is_deleted = FALSE"}
${options.last_known_tiddler_id ? "AND tiddler_id > $last_known_tiddler_id" : ""}
GROUP BY t.title
ORDER BY t.title, tiddler_id DESC
${options.limit ? "LIMIT $limit" : ""}
)
`,params);
return rows;
};
/*
Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist
*/
SqlTiddlerDatabase.prototype.getRecipeLastTiddlerId = function(recipe_name) {
const row = this.engine.runStatementGet(`
SELECT t.title, t.tiddler_id, b.bag_name, MAX(rb.position) AS position
FROM bags AS b
INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id
INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
WHERE r.recipe_name = $recipe_name
GROUP BY t.title
ORDER BY t.tiddler_id DESC
LIMIT 1
`,{
$recipe_name: recipe_name
});
if(row) {
return row.tiddler_id;
} else {
return null;
}
};
SqlTiddlerDatabase.prototype.deleteAllTiddlersInBag = function(bag_name) {
// Delete the fields
this.engine.runStatement(`

18
plugins/tiddlywiki/multiwikiserver/modules/store/sql-tiddler-store.js
View File

@ -306,11 +306,25 @@ SqlTiddlerStore.prototype.getBagTiddlers = function(bag_name) {
return this.sqlTiddlerDatabase.getBagTiddlers(bag_name);
};
/*
Get the tiddler_id of the newest tiddler in a bag. Returns null for bags that do not exist
*/
SqlTiddlerStore.prototype.getBagLastTiddlerId = function(bag_name) {
return this.sqlTiddlerDatabase.getBagLastTiddlerId(bag_name);
};
/*
Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist
*/
SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name) {
return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name);
SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipe_name,options) {
return this.sqlTiddlerDatabase.getRecipeTiddlers(recipe_name,options);
};
/*
Get the tiddler_id of the newest tiddler in a recipe. Returns null for recipes that do not exist
*/
SqlTiddlerStore.prototype.getRecipeLastTiddlerId = function(recipe_name) {
return this.sqlTiddlerDatabase.getRecipeLastTiddlerId(recipe_name);
};
SqlTiddlerStore.prototype.deleteAllTiddlersInBag = function(bag_name) {

20
plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-database.js
View File

@ -74,12 +74,12 @@ function runSqlDatabaseTests(engine) {
});
// Verify what we've got
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
{ title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta' }
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
{ title: 'Hello There', tiddler_id: 3, bag_name: 'bag-beta', is_deleted: 0 }
]);
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' }
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 }
]);
expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-rho").tiddler).toEqual({ title: "Hello There", text: "I'm in beta", tags: "four five six" });
expect(sqlTiddlerDatabase.getRecipeTiddler("Missing Tiddler","recipe-rho")).toEqual(null);
@ -90,17 +90,17 @@ function runSqlDatabaseTests(engine) {
// Delete a tiddlers to ensure the underlying tiddler in the recipe shows through
sqlTiddlerDatabase.deleteTiddler("Hello There","bag-beta");
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
{ title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' }
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
{ title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 }
]);
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha' },
{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' }
{ title: 'Another Tiddler', tiddler_id: 1, bag_name: 'bag-alpha', is_deleted: 0 },
{ title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 }
]);
expect(sqlTiddlerDatabase.getRecipeTiddler("Hello There","recipe-beta")).toEqual(null);
sqlTiddlerDatabase.deleteTiddler("Another Tiddler","bag-alpha");
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha' } ]);
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma' } ]);
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-rho")).toEqual([ { title: 'Hello There', tiddler_id: 2, bag_name: 'bag-alpha', is_deleted: 0 } ]);
expect(sqlTiddlerDatabase.getRecipeTiddlers("recipe-sigma")).toEqual([ { title: 'Hello There', tiddler_id: 4, bag_name: 'bag-gamma', is_deleted: 0 } ]);
// Save a recipe tiddler
expect(sqlTiddlerDatabase.saveRecipeTiddler({title: "More", text: "None"},"recipe-rho")).toEqual({tiddler_id: 7, bag_name: 'bag-beta'});
expect(sqlTiddlerDatabase.getRecipeTiddler("More","recipe-rho").tiddler).toEqual({title: "More", text: "None"});

2
plugins/tiddlywiki/multiwikiserver/modules/store/tests-sql-tiddler-store.js
View File

@ -123,7 +123,7 @@ function runSqlStoreTests(engine) {
expect(typeof(saveRecipeResult.tiddler_id)).toBe("number");
expect(saveRecipeResult.bag_name).toBe("bag-beta");
expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta"}]);
expect(store.getRecipeTiddlers("recipe-rho")).toEqual([{title: "Another Tiddler", tiddler_id: 1, bag_name: "bag-beta", is_deleted: 0 }]);
var getRecipeTiddlerResult = store.getRecipeTiddler("Another Tiddler","recipe-rho");
expect(typeof(getRecipeTiddlerResult.tiddler_id)).toBe("number");