diff --git a/boot/boot.js b/boot/boot.js index df89561d46..fb195bed6d 100644 --- a/boot/boot.js +++ b/boot/boot.js @@ -1971,7 +1971,7 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) { }); // Helper to process a file - var processFile = function(filename,isTiddlerFile,fields,isEditableFile,rootPath) { + var processFile = function(filename,isTiddlerFile,fields,isEditableFile,rootPath,dynamicStoreId) { var extInfo = $tw.config.fileExtensionInfo[path.extname(filename)], type = (extInfo || {}).type || fields.type || "text/plain", typeInfo = $tw.config.contentTypeInfo[type] || {}, @@ -2046,9 +2046,9 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) { }); }); if(isEditableFile) { - tiddlers.push({filepath: pathname, hasMetaFile: !!metadata && !isTiddlerFile, isEditableFile: true, tiddlers: fileTiddlers}); + tiddlers.push({filepath: pathname, hasMetaFile: !!metadata && !isTiddlerFile, isEditableFile: true, dynamicStoreId: dynamicStoreId, tiddlers: fileTiddlers}); } else { - tiddlers.push({tiddlers: fileTiddlers}); + tiddlers.push({dynamicStoreId: dynamicStoreId, tiddlers: fileTiddlers}); } }; // Helper to recursively search subdirectories @@ -2089,6 +2089,31 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) { // Process directory specifier var dirPath = path.resolve(filepath,dirSpec.path); if(fs.existsSync(dirPath) && fs.statSync(dirPath).isDirectory()) { + // Register a dynamic store if requested + var dynamicStoreId = null; + if(dirSpec.dynamicStore && $tw.boot.dynamicStores) { + dynamicStoreId = dirPath; + var existing = null; + for(var ds=0; ds<$tw.boot.dynamicStores.length; ds++) { + if($tw.boot.dynamicStores[ds].id === dynamicStoreId) { + existing = $tw.boot.dynamicStores[ds]; + break; + } + } + if(!existing) { + $tw.boot.dynamicStores.push({ + id: dynamicStoreId, + directory: dirPath, + saveFilter: dirSpec.dynamicStore.saveFilter || "", + watch: dirSpec.dynamicStore.watch !== false, + debounce: dirSpec.dynamicStore.debounce || 400, + filesRegExp: dirSpec.filesRegExp || "^.*$", + searchSubdirectories: !!dirSpec.searchSubdirectories, + isTiddlerFile: !!dirSpec.isTiddlerFile, + fields: dirSpec.fields || {} + }); + } + } var files = getAllFiles(dirPath, dirSpec.searchSubdirectories), fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"), metaRegExp = /^.*\.meta$/; @@ -2097,7 +2122,7 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) { filename = path.basename(thisPath); if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) { dirSpec.fields = dirSpec.fields || {}; - processFile(thisPath,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile,dirSpec.path); + processFile(thisPath,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile || !!dirSpec.dynamicStore,dirSpec.path,dynamicStoreId); } } } else { @@ -2293,7 +2318,8 @@ $tw.loadWikiTiddlers = function(wikiPath,options) { filepath: tiddlerFile.filepath, type: tiddlerFile.type, hasMetaFile: tiddlerFile.hasMetaFile, - isEditableFile: config["retain-original-tiddler-path"] || tiddlerFile.isEditableFile || tiddlerFile.filepath.indexOf($tw.boot.wikiTiddlersPath) !== 0 + isEditableFile: config["retain-original-tiddler-path"] || tiddlerFile.isEditableFile || tiddlerFile.filepath.indexOf($tw.boot.wikiTiddlersPath) !== 0, + dynamicStoreId: tiddlerFile.dynamicStoreId || null }; }); } @@ -2431,6 +2457,8 @@ $tw.boot.initStartup = function(options) { if(!$tw.boot.tasks.readBrowserTiddlers) { // For writable tiddler files, a hashmap of title to {filepath:,type:,hasMetaFile:} $tw.boot.files = Object.create(null); + // Array of {id, directory, saveFilter, watch, debounce} registered via tiddlywiki.files dynamicStore directives + $tw.boot.dynamicStores = []; // System paths and filenames $tw.boot.bootPath = options.bootPath || path.dirname(module.filename); $tw.boot.corePath = path.resolve($tw.boot.bootPath,"../core"); diff --git a/editions/test/tiddlers/TabFour.tid b/editions/test/tiddlers/TabFour.tid new file mode 100644 index 0000000000..f94963c3cd --- /dev/null +++ b/editions/test/tiddlers/TabFour.tid @@ -0,0 +1,4 @@ +modified: 20260413092032887 +title: TabFour + +Text tab 4 \ No newline at end of file diff --git a/editions/test/tiddlers/TabOne.tid b/editions/test/tiddlers/TabOne.tid new file mode 100644 index 0000000000..ddc44d649a --- /dev/null +++ b/editions/test/tiddlers/TabOne.tid @@ -0,0 +1,5 @@ +caption: t 1 +modified: 20260413092032887 +title: TabOne + +Text tab 1 \ No newline at end of file diff --git a/editions/test/tiddlers/TabThree.tid b/editions/test/tiddlers/TabThree.tid new file mode 100644 index 0000000000..30b660c568 --- /dev/null +++ b/editions/test/tiddlers/TabThree.tid @@ -0,0 +1,6 @@ +caption: t 3 +description: desc +modified: 20260413092032887 +title: TabThree + +Text tab 3 \ No newline at end of file diff --git a/editions/test/tiddlers/TabTwo.tid b/editions/test/tiddlers/TabTwo.tid new file mode 100644 index 0000000000..1f4d502488 --- /dev/null +++ b/editions/test/tiddlers/TabTwo.tid @@ -0,0 +1,5 @@ +caption: t 2 +modified: 20260413092032887 +title: TabTwo + +Text tab 2 \ No newline at end of file diff --git a/editions/test/tiddlers/body-template.tid b/editions/test/tiddlers/body-template.tid new file mode 100644 index 0000000000..c5633c0e60 --- /dev/null +++ b/editions/test/tiddlers/body-template.tid @@ -0,0 +1,7 @@ +code-body: yes +modified: 20260413092032887 +title: body-template + +!! <> + +<$transclude tiddler=<> mode="block"/> \ No newline at end of file diff --git a/editions/test/tiddlers/button-template.tid b/editions/test/tiddlers/button-template.tid new file mode 100644 index 0000000000..9723f93a1f --- /dev/null +++ b/editions/test/tiddlers/button-template.tid @@ -0,0 +1,5 @@ +code-body: yes +modified: 20260413092032887 +title: button-template + +<$transclude tiddler=<> field="description"><$transclude tiddler=<> field="caption"><$macrocall $name="currentTab" $type="text/plain" $output="text/plain"/> \ No newline at end of file diff --git a/editions/test/tiddlers/tabs-macro-definition.tid b/editions/test/tiddlers/tabs-macro-definition.tid new file mode 100644 index 0000000000..00798f6aee --- /dev/null +++ b/editions/test/tiddlers/tabs-macro-definition.tid @@ -0,0 +1,72 @@ +code-body: yes +modified: 20260413092032887 +title: tabs-macro-definition + +\define tabs-button() +\whitespace trim +<$button + set=<> + setTo=<> + default=<<__default__>> + selectedClass="tc-tab-selected" + selectedAria="aria-selected" + tooltip={{!!tooltip}} + role="tab" + data-tab-title=<> +> + <$tiddler tiddler=<>> + <$set name="tv-wikilinks" value="no"> + <$transclude tiddler=<<__buttonTemplate__>> mode="inline"> + <$transclude tiddler=<> field="caption"> + <$macrocall $name="currentTab" $type="text/plain" $output="text/plain"/> + + + + + <<__actions__>> + +\end + +\define tabs-tab() +\whitespace trim +<$set name="save-currentTiddler" value=<>> + <$tiddler tiddler=<>> + <> + + +\end + +\define tabs-tab-list() +\whitespace trim +<$list filter=<<__tabsList__>> variable="currentTab" storyview="pop"> + <> + +\end + +\define tabs-tab-body() +\whitespace trim +<$list filter=<<__tabsList__>> variable="currentTab"> + <$reveal type="match" state=<> text=<> default=<<__default__>> retain=<<__retain__>> tag="div"> + <$transclude tiddler=<<__template__>> mode="block"> + <$transclude tiddler=<> mode="block"/> + + + +\end + +\define tabs(tabsList,default,state:"$:/state/tab",class,template,buttonTemplate,retain,actions,explicitState) +\whitespace trim +<$qualify title=<<__state__>> name="qualifiedState"> + <$let tabsState={{{ [<__explicitState__>minlength[1]] ~[] }}}> +
] }}} role="tablist"> +
] }}}> + <> +
+
] }}}/> +
] }}} role="tabpanel"> + <> +
+
+ + +\end \ No newline at end of file diff --git a/editions/test/tiddlers/test-tabs-horizontal-all.tid b/editions/test/tiddlers/test-tabs-horizontal-all.tid new file mode 100644 index 0000000000..53c122d9e1 --- /dev/null +++ b/editions/test/tiddlers/test-tabs-horizontal-all.tid @@ -0,0 +1,5 @@ +modified: 20260413092032887 +title: test-tabs-horizontal-all + +\import [[tabs-macro-definition]] +<> \ No newline at end of file diff --git a/editions/test/tiddlers/test-tabs-horizontal.tid b/editions/test/tiddlers/test-tabs-horizontal.tid new file mode 100644 index 0000000000..42aec8b60b --- /dev/null +++ b/editions/test/tiddlers/test-tabs-horizontal.tid @@ -0,0 +1,5 @@ +modified: 20260413092032887 +title: test-tabs-horizontal + +\import [[tabs-macro-definition]] +<> \ No newline at end of file diff --git a/editions/test/tiddlers/test-tabs-vertical.tid b/editions/test/tiddlers/test-tabs-vertical.tid new file mode 100644 index 0000000000..05bce132ec --- /dev/null +++ b/editions/test/tiddlers/test-tabs-vertical.tid @@ -0,0 +1,5 @@ +modified: 20260413092032887 +title: test-tabs-vertical + +\import [[tabs-macro-definition]] +<> \ No newline at end of file diff --git a/editions/test/tiddlers/tests/test-filesystem-dynamic-store.js b/editions/test/tiddlers/tests/test-filesystem-dynamic-store.js new file mode 100644 index 0000000000..ee9fd74672 --- /dev/null +++ b/editions/test/tiddlers/tests/test-filesystem-dynamic-store.js @@ -0,0 +1,206 @@ +/*\ +title: test-filesystem-dynamic-store.js +type: application/javascript +tags: [[$:/tags/test-spec]] + +Tests for the filesystem syncadaptor dynamic store feature: save routing +driven by saveFilter, and chokidar-based watching of out-of-band edits. + +\*/ +"use strict"; + +if($tw.node) { + + var fs = require("fs"), + path = require("path"), + os = require("os"); + + // Load the filesystem adaptor source as if it were a TW module, so that + // $tw is provided without having to include the plugin in the edition + // (which would pull in the server-side syncer and keep the test runner alive). + var adaptorPath = path.resolve($tw.boot.bootPath,"..","plugins","tiddlywiki","filesystem","filesystemadaptor.js"), + adaptorTitle = "$:/plugins/tiddlywiki/filesystem/filesystemadaptor.js"; + if(!$tw.modules.titles[adaptorTitle]) { + $tw.modules.titles[adaptorTitle] = { + moduleType: "syncadaptor", + definition: fs.readFileSync(adaptorPath,"utf8") + }; + $tw.wiki.addTiddler({ + title: adaptorTitle, + type: "application/javascript", + "module-type": "syncadaptor", + text: "" + }); + } + var FileSystemAdaptor = $tw.modules.execute(adaptorTitle).adaptorClass; + + function makeTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(),prefix)); + } + + function removeDirRecursive(dir) { + if(fs.existsSync(dir)) { + fs.rmSync(dir,{recursive: true, force: true}); + } + } + + function waitMs(ms) { + return new Promise(function(resolve) { setTimeout(resolve,ms); }); + } + + describe("filesystem dynamic store", function() { + + var tmpRoot, wikiTiddlers, storeDir, origDynamicStores, origFiles, originalBootPath; + var adaptor, wiki; + + beforeEach(function() { + tmpRoot = makeTempDir("tw-dyn-"); + wikiTiddlers = path.join(tmpRoot,"tiddlers"); + storeDir = path.join(tmpRoot,"content"); + fs.mkdirSync(wikiTiddlers); + fs.mkdirSync(storeDir); + + origDynamicStores = $tw.boot.dynamicStores; + origFiles = $tw.boot.files; + originalBootPath = $tw.boot.wikiTiddlersPath; + + $tw.boot.dynamicStores = [{ + id: storeDir, + directory: storeDir, + saveFilter: "[type[text/x-markdown]]", + watch: true, + debounce: 40, + filesRegExp: ".*\\.tid$", + searchSubdirectories: false, + isTiddlerFile: true, + fields: {} + }]; + $tw.boot.files = Object.create(null); + $tw.boot.wikiTiddlersPath = wikiTiddlers; + + wiki = new $tw.Wiki(); + adaptor = new FileSystemAdaptor({wiki: wiki, boot: $tw.boot}); + }); + + afterEach(function(done) { + adaptor.close().then(function() { + $tw.boot.dynamicStores = origDynamicStores; + $tw.boot.files = origFiles; + $tw.boot.wikiTiddlersPath = originalBootPath; + removeDirRecursive(tmpRoot); + done(); + }); + }); + + it("routes saves for matching tiddlers into the dynamic store directory", function(done) { + wiki.addTiddler(new $tw.Tiddler({title: "note1", type: "text/x-markdown", text: "hello"})); + var tiddler = wiki.getTiddler("note1"); + adaptor.getTiddlerFileInfo(tiddler,function(err,fileInfo) { + expect(err).toBeFalsy(); + expect(fileInfo.filepath.indexOf(storeDir)).toBe(0); + expect(fileInfo.dynamicStoreId).toBe(storeDir); + done(); + }); + }); + + it("routes saves for non-matching tiddlers into the default wiki tiddlers path", function(done) { + wiki.addTiddler(new $tw.Tiddler({title: "note2", type: "text/vnd.tiddlywiki", text: "plain"})); + var tiddler = wiki.getTiddler("note2"); + adaptor.getTiddlerFileInfo(tiddler,function(err,fileInfo) { + expect(err).toBeFalsy(); + expect(fileInfo.filepath.indexOf(wikiTiddlers)).toBe(0); + expect(fileInfo.dynamicStoreId).toBeFalsy(); + done(); + }); + }); + + it("keeps saving a tiddler into the store it originally came from", function(done) { + // Simulate a tiddler that was loaded at boot from the dynamic store + $tw.boot.files["frozen"] = { + filepath: path.join(storeDir,"frozen.tid"), + type: "application/x-tiddler", + hasMetaFile: false, + isEditableFile: true, + dynamicStoreId: storeDir + }; + // Its current type no longer matches the saveFilter — store id must still win + wiki.addTiddler(new $tw.Tiddler({title: "frozen", type: "text/vnd.tiddlywiki", text: "still here"})); + adaptor.getTiddlerFileInfo(wiki.getTiddler("frozen"),function(err,fileInfo) { + expect(err).toBeFalsy(); + expect(fileInfo.filepath.indexOf(storeDir)).toBe(0); + expect(fileInfo.dynamicStoreId).toBe(storeDir); + done(); + }); + }); + + it("detects out-of-band additions and changes via chokidar", function(done) { + if(adaptor.watchers.length === 0) { + // chokidar unavailable in this environment + pending("chokidar not installed"); + return done(); + } + var filepath = path.join(storeDir,"external.tid"); + // Write initial file + fs.writeFileSync(filepath,"title: external\ntype: text/x-markdown\n\nInitial\n"); + waitMs(200).then(function() { + return new Promise(function(resolve) { + adaptor.getUpdatedTiddlers({},function(err,updates) { + expect(err).toBeFalsy(); + expect(updates.modifications).toContain("external"); + resolve(); + }); + }); + }).then(function() { + // loadTiddler should now return the fields from disk + return new Promise(function(resolve) { + adaptor.loadTiddler("external",function(err,fields) { + expect(err).toBeFalsy(); + expect(fields).toBeTruthy(); + expect(fields.title).toBe("external"); + expect(fields.text).toContain("Initial"); + resolve(); + }); + }); + }).then(function() { + // Edit the file + fs.writeFileSync(filepath,"title: external\ntype: text/x-markdown\n\nChanged\n"); + return waitMs(200); + }).then(function() { + return new Promise(function(resolve) { + adaptor.getUpdatedTiddlers({},function(err,updates) { + expect(updates.modifications).toContain("external"); + resolve(); + }); + }); + }).then(function() { + // Delete the file + fs.unlinkSync(filepath); + return waitMs(200); + }).then(function() { + return new Promise(function(resolve) { + adaptor.getUpdatedTiddlers({},function(err,updates) { + expect(updates.deletions).toContain("external"); + resolve(); + }); + }); + }).then(done, done.fail); + }); + + it("suppresses echoes when the file on disk matches the current wiki tiddler", function(done) { + if(adaptor.watchers.length === 0) { + pending("chokidar not installed"); + return done(); + } + // Seed the wiki with a tiddler whose content exactly matches what we'll write + wiki.addTiddler(new $tw.Tiddler({title: "echo", type: "text/x-markdown", text: "same\n"})); + var filepath = path.join(storeDir,"echo.tid"); + fs.writeFileSync(filepath,"title: echo\ntype: text/x-markdown\n\nsame\n"); + waitMs(200).then(function() { + adaptor.getUpdatedTiddlers({},function(err,updates) { + expect(updates.modifications).not.toContain("echo"); + done(); + }); + }); + }); + }); +} diff --git a/editions/tw5.com/tiddlers/nodejs/tiddlywiki.files_Files.tid b/editions/tw5.com/tiddlers/nodejs/tiddlywiki.files_Files.tid index b613317d37..96777f80df 100644 --- a/editions/tw5.com/tiddlers/nodejs/tiddlywiki.files_Files.tid +++ b/editions/tw5.com/tiddlers/nodejs/tiddlywiki.files_Files.tid @@ -56,6 +56,11 @@ Directory specifications in the `directories` array may take the following forms ** ''isEditableFile'' - <<.from-version "5.1.23">> (optional) if `true`, changes to the tiddler be saved back to the original file. The tiddler will be saved back to the original filepath as long as it does not generate a result from the $:/config/FileSystemPath filters, which will override the final filepath generated if a result is returned from a filter. ** ''searchSubdirectories'' - <<.from-version "5.1.23">> (optional) if `true`, all subdirectories of the //path// are searched recursively for files that match the (optional) //filesRegExp//. If no //filesRegExp// is provided, all files in all subdirectories of the //path// are loaded. Tiddler titles generated via the //source// attribute //filename// (see above) will only include the filename, not any of the subdirectories of the path. If this results in multiple files with loaded with the same tiddler title, then only the last file loaded under that tiddler title will be in memory. In order to prevent this, you can use the //filepath// attribute instead of //filename//. Alternately, you can include multiple directory objects and customise the title field with a //prefix// or //suffix// alongside the //source// attribute. ** ''fields'' - (required) an object containing values that override or customise the fields provided in the tiddler file (see above) +** ''dynamicStore'' - <<.from-version "5.5.0">> (optional) an object marking the directory as a //dynamic store// that is both loaded at boot and actively watched on disk. The filesystem syncadaptor uses the watcher to pick up out-of-band changes (e.g. edits made by an external editor) and folds them into the running wiki. The object has the following properties: +*** ''saveFilter'' - (optional) a filter evaluated against each tiddler the wiki tries to save. Tiddlers that match are saved back into this directory instead of the default tiddlers folder. The first matching dynamic store wins, so specificity matters when multiple stores are registered. +*** ''watch'' - (optional, defaults to `true`) set to `false` to disable the chokidar watcher for this store (tiddlers are still loaded and save-routed, but external changes are not picked up live) +*** ''debounce'' - (optional, defaults to `400`) the per-file debounce window in milliseconds. File events that occur within this window of a previous event for the same file are coalesced, which avoids duplicated reloads during atomic-rename saves performed by many editors. +*** Changes to a file on disk are diffed against the tiddler currently in the wiki before being reported, so self-writes performed by TiddlyWiki itself do not trigger spurious reload events. JavaScript module tiddlers (tiddlers with `type: application/javascript` and a `module-type` field) are never hot-reloaded, because that would require restarting the TiddlyWiki process. Fields can also be overridden for particular files by creating a file with the same name plus the suffix `.meta` -- see TiddlerFiles. @@ -135,6 +140,29 @@ From the examples in [[Customising Tiddler File Naming]] we see that the final ` Then, the `[tag[.txt]then[.txt]]` filter in the $:/config/FileSystemExtensions tiddler forces all these tiddlers to be saved back to disk as *.txt and accompanying *.txt.meta files (overriding the normal tiddler-type to file-type mapping). In this case, allowing the snippets of Tiddlywiki wikitext or markdown-text to be saved back to "text" *.txt files. +!! Dynamic store for markdown files + +<<.from-version "5.5.0">> This example treats a sibling `content/` folder as a dynamic store: every `.md` file is loaded at boot, every markdown tiddler saved by the wiki is routed back into `content/`, and external edits to those files are picked up automatically by chokidar. + +``` +{ + "directories": [ + { + "path": "../content", + "filesRegExp": "^.*\\.md$", + "isTiddlerFile": true, + "searchSubdirectories": true, + "dynamicStore": { + "saveFilter": "[type[text/x-markdown]]", + "watch": true, + "debounce": 400 + }, + "fields": {} + } + ] +} +``` + !! Importing and auto-tagging images This example imports all the image files in the `files` directory and all its subdirectories as external-image tiddlers, and tags them based on their filepath. Each tiddler is set up with the following fields: diff --git a/editions/tw5.com/tiddlers/releasenotes/5.5.0/Release 5.5.0.tid b/editions/tw5.com/tiddlers/releasenotes/5.5.0/Release 5.5.0.tid new file mode 100644 index 0000000000..14c807a369 --- /dev/null +++ b/editions/tw5.com/tiddlers/releasenotes/5.5.0/Release 5.5.0.tid @@ -0,0 +1,13 @@ +caption: 5.5.0 +created: 20260413000000000 +modified: 20260413000000000 +tags: ReleaseNotes +title: Release 5.5.0 +type: text/vnd.tiddlywiki +description: Under development + +\procedure release-introduction() +Release v5.5.0 is under development. +\end release-introduction + +<> diff --git a/editions/tw5.com/tiddlers/releasenotes/5.5.0/dynamic-store.tid b/editions/tw5.com/tiddlers/releasenotes/5.5.0/dynamic-store.tid new file mode 100644 index 0000000000..b4b2166827 --- /dev/null +++ b/editions/tw5.com/tiddlers/releasenotes/5.5.0/dynamic-store.tid @@ -0,0 +1,34 @@ +title: $:/changenotes/5.5.0/dynamic-store +description: Filesystem dynamic stores with live reload via chokidar +release: 5.5.0 +tags: $:/tags/ChangeNote +change-type: feature +change-category: nodejs + +The filesystem syncadaptor can now be configured to treat a folder as a [[dynamic store|tiddlywiki.files Files]]: tiddlers in the folder are loaded at boot, a configurable filter decides which tiddlers are saved back into that folder instead of the default `tiddlers/` directory, and external edits to files in the folder are picked up live by a [[chokidar|https://github.com/paulmillr/chokidar]] watcher. + +Dynamic stores are declared inside a `tiddlywiki.files` specification via the new `dynamicStore` property on a directory entry. For example, the following declaration loads every `.md` file in a sibling `content/` folder, routes any tiddler with `type: text/x-markdown` back to that folder on save, and hot-reloads changes made by an external editor: + +``` +{ + "directories": [{ + "path": "../content", + "filesRegExp": "^.*\\.md$", + "isTiddlerFile": true, + "dynamicStore": { + "saveFilter": "[type[text/x-markdown]]", + "watch": true, + "debounce": 400 + } + }] +} +``` + +Notes: + +* File events are debounced per file (default 400ms, configurable via `debounce`) to cope with editors that save atomically via rename. +* Each detected change is diffed against the current in-wiki tiddler before being reported, so self-writes performed by TiddlyWiki itself do not cause reload loops. +* Deletions on disk propagate to the wiki via the syncer's standard server-side-deletion path. +* JavaScript module tiddlers (those with `type: application/javascript` and a `module-type` field) are never hot-reloaded; reloading them would require restarting the TiddlyWiki process. + +This feature adds [[chokidar|https://github.com/paulmillr/chokidar]] as a new runtime dependency of TiddlyWiki on Node.js. diff --git a/package-lock.json b/package-lock.json index 2382ebedcf..7581d63156 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,9 @@ "name": "tiddlywiki", "version": "5.4.0-prerelease", "license": "BSD", + "dependencies": { + "chokidar": "^4.0.3" + }, "bin": { "tiddlywiki": "tiddlywiki.js" }, @@ -19,7 +22,7 @@ "globals": "16.4.0" }, "engines": { - "node": ">=0.8.2" + "node": ">=20.0.0" } }, "node_modules/@eslint-community/eslint-utils": { @@ -329,6 +332,21 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/color-convert": { "version": "2.0.1", "dev": true, @@ -898,6 +916,19 @@ "node": ">=6" } }, + "node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/resolve-from": { "version": "4.0.0", "dev": true, diff --git a/package.json b/package.json index ee56f5d029..fdb2fa3b58 100644 --- a/package.json +++ b/package.json @@ -23,11 +23,14 @@ "tiddlywiki5", "wiki" ], + "dependencies": { + "chokidar": "^4.0.3" + }, "devDependencies": { "@eslint/js": "9.36.0", + "@stylistic/eslint-plugin": "5.4.0", "eslint": "9.36.0", "eslint-plugin-es-x": "9.1.0", - "@stylistic/eslint-plugin": "5.4.0", "globals": "16.4.0" }, "license": "BSD", diff --git a/plugins/tiddlywiki/filesystem/filesystemadaptor.js b/plugins/tiddlywiki/filesystem/filesystemadaptor.js index 76f6fa7215..246933e893 100644 --- a/plugins/tiddlywiki/filesystem/filesystemadaptor.js +++ b/plugins/tiddlywiki/filesystem/filesystemadaptor.js @@ -11,6 +11,7 @@ A sync adaptor module for synchronising with the local filesystem via node.js AP // Get a reference to the file system var fs = $tw.node ? require("fs") : null; +var path = $tw.node ? require("path") : null; function FileSystemAdaptor(options) { this.wiki = options.wiki; @@ -20,6 +21,12 @@ function FileSystemAdaptor(options) { if(this.boot.wikiTiddlersPath) { $tw.utils.createDirectory(this.boot.wikiTiddlersPath); } + // Buffers for out-of-band file changes, drained by getUpdatedTiddlers + this.modifications = Object.create(null); + this.deletions = Object.create(null); + this.pendingTimers = Object.create(null); + this.watchers = []; + this.setupWatchers(); } FileSystemAdaptor.prototype.name = "filesystem"; @@ -27,35 +34,56 @@ FileSystemAdaptor.prototype.name = "filesystem"; FileSystemAdaptor.prototype.supportsLazyLoading = false; FileSystemAdaptor.prototype.isReady = function() { - // The file system adaptor is always ready return true; }; FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) { - //Returns the existing fileInfo for the tiddler. To regenerate, call getTiddlerFileInfo(). var title = tiddler.fields.title; return this.boot.files[title]; }; /* -Return a fileInfo object for a tiddler, creating it if necessary: - filepath: the absolute path to the file containing the tiddler - type: the type of the tiddler file (NOT the type of the tiddler -- see below) - hasMetaFile: true if the file also has a companion .meta file +Find the dynamic store (if any) that a tiddler should be saved into. +Precedence: existing boot.files entry wins; otherwise first matching saveFilter. +*/ +FileSystemAdaptor.prototype.findDynamicStoreForTiddler = function(tiddler) { + var stores = this.boot.dynamicStores || []; + if(stores.length === 0) { + return null; + } + var title = tiddler.fields.title, + existing = this.boot.files[title]; + if(existing && existing.dynamicStoreId) { + for(var i=0; i 0) { + return store; + } + } + } + return null; +}; -The boot process populates this.boot.files for each of the tiddler files that it loads. -The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files). - -It is the responsibility of the filesystem adaptor to update this.boot.files for new files that are created. +/* +Return a fileInfo object for a tiddler, creating it if necessary. */ FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) { - // Error if we don't have a this.boot.wikiTiddlersPath if(!this.boot.wikiTiddlersPath) { return callback("filesystemadaptor requires a valid wiki folder"); } - // Always generate a fileInfo object when this fuction is called var title = tiddler.fields.title, newInfo, pathFilters, extFilters, - fileInfo = this.boot.files[title]; + fileInfo = this.boot.files[title], + store = this.findDynamicStoreForTiddler(tiddler), + directory = store ? store.directory : this.boot.wikiTiddlersPath; if(this.wiki.tiddlerExists("$:/config/FileSystemPaths")) { pathFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"); } @@ -63,12 +91,15 @@ FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) { extFilters = this.wiki.getTiddlerText("$:/config/FileSystemExtensions","").split("\n"); } newInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{ - directory: this.boot.wikiTiddlersPath, + directory: directory, pathFilters: pathFilters, extFilters: extFilters, wiki: this.wiki, fileInfo: fileInfo }); + if(store) { + newInfo.dynamicStoreId = store.id; + } callback(null,newInfo); }; @@ -83,6 +114,7 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback,options) { if(err) { return callback(err); } + var dynamicStoreId = fileInfo && fileInfo.dynamicStoreId || null; $tw.utils.saveTiddlerToFile(tiddler,fileInfo,function(err,fileInfo) { if(err) { if((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "open") { @@ -95,6 +127,9 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback,options) { return callback(err); } } + if(dynamicStoreId && fileInfo) { + fileInfo.dynamicStoreId = dynamicStoreId; + } // Store new boot info only after successful writes self.boot.files[tiddler.fields.title] = fileInfo; // Cleanup duplicates if the file moved or changed extensions @@ -116,9 +151,28 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback,options) { /* Load a tiddler and invoke the callback with (err,tiddlerFields) -We don't need to implement loading for the file system adaptor, because all the tiddler files will have been loaded during the boot process. +Most tiddlers are pre-loaded at boot, but the syncer may ask us to load +individual tiddlers in response to watcher-driven out-of-band changes. */ FileSystemAdaptor.prototype.loadTiddler = function(title,callback) { + var fileInfo = this.boot.files[title]; + if(!fileInfo || !fileInfo.dynamicStoreId || !fs.existsSync(fileInfo.filepath)) { + return callback(null,null); + } + var loaded; + try { + loaded = $tw.loadTiddlersFromFile(fileInfo.filepath,{}); + } catch(e) { + return callback(e); + } + if(!loaded || !loaded.tiddlers) { + return callback(null,null); + } + for(var i=0; i 0) { + this.logger.log("Dynamic store: detected removal of " + deletedTitles.length + " tiddler(s) at " + filepath); + } + return; + } + // Add/change: re-parse the file and queue modifications + var loaded; + try { + loaded = $tw.loadTiddlersFromFile(filepath,{}); + } catch(e) { + this.logger.log("Failed to load tiddler file " + filepath,e.message); + return; + } + if(!loaded || !loaded.tiddlers) { + return; + } + var newTitles = {}; + loaded.tiddlers.forEach(function(fields) { + if(!fields || !fields.title) { + return; + } + if(fields.type === "application/javascript" && fields["module-type"]) { + self.logger.log("Skipping hot-reload of JS module tiddler " + fields.title + " (requires a restart)"); + return; + } + var title = fields.title; + newTitles[title] = true; + // Ensure boot.files tracks the file so loadTiddler can find it on demand + self.boot.files[title] = { + filepath: loaded.filepath, + type: loaded.type, + hasMetaFile: loaded.hasMetaFile, + isEditableFile: true, + dynamicStoreId: store.id + }; + // Diff against the current wiki tiddler to suppress self-write echoes + var existing = self.wiki.getTiddler(title); + if(existing && self.tiddlerFieldsEqual(existing.fields,fields)) { + return; + } + self.modifications[title] = true; + delete self.deletions[title]; + }); + // Handle tiddlers that were previously in this file but have now disappeared + $tw.utils.each(this.boot.files,function(info,title) { + if(info && info.filepath === filepath && !newTitles[title]) { + delete self.boot.files[title]; + self.deletions[title] = true; + delete self.modifications[title]; + } + }); +}; + +FileSystemAdaptor.prototype.tiddlerFieldsEqual = function(existingFields,newFields) { + // Ignore volatile fields that the syncer / server may add + var ignore = {revision: 1, bag: 1}; + var keys = {}; + $tw.utils.each(existingFields,function(v,k) { if(!ignore[k]) keys[k] = true; }); + $tw.utils.each(newFields,function(v,k) { if(!ignore[k]) keys[k] = true; }); + for(var k in keys) { + var a = existingFields[k], + b = newFields[k]; + // Normalise arrays to string form + if($tw.utils.isArray(a)) a = $tw.utils.stringifyList(a); + if($tw.utils.isArray(b)) b = $tw.utils.stringifyList(b); + if(a instanceof Date) a = $tw.utils.stringifyDate(a); + if(b instanceof Date) b = $tw.utils.stringifyDate(b); + if((a === undefined ? "" : String(a)) !== (b === undefined ? "" : String(b))) { + return false; + } + } + return true; }; if(fs) {