From 373afd72c804b0ffa2779849ba0d278d53e65df4 Mon Sep 17 00:00:00 2001 From: Jermolene Date: Sun, 14 Apr 2019 12:04:00 +0100 Subject: [PATCH] Add savewikifolder command Makes it much easier to convert a TiddlyWiki HTML file into a full wiki folder. --- core/language/en-GB/Help/savewikifolder.tid | 19 ++ core/modules/commands/savewikifolder.js | 190 ++++++++++++++++++ core/modules/utils/filesystem.js | 41 +++- .../commands/SaveWikiFolderCommand.tid | 7 + 4 files changed, 248 insertions(+), 9 deletions(-) create mode 100644 core/language/en-GB/Help/savewikifolder.tid create mode 100644 core/modules/commands/savewikifolder.js create mode 100644 editions/tw5.com/tiddlers/commands/SaveWikiFolderCommand.tid diff --git a/core/language/en-GB/Help/savewikifolder.tid b/core/language/en-GB/Help/savewikifolder.tid new file mode 100644 index 000000000..2f6f034f1 --- /dev/null +++ b/core/language/en-GB/Help/savewikifolder.tid @@ -0,0 +1,19 @@ +title: $:/language/Help/savewikifolder +description: Saves a wiki to a new wiki folder + +Saves the current wiki as a wiki folder, including tiddlers, plugins and configuration: + +``` +--savewikifolder [] +``` + +* The target wiki folder must be empty or non-existent +* The filter specifies which tiddlers should be included. It is optional, defaulting to `[all[tiddlers]]` +* Plugins from the official plugin library are replaced with references to those plugins in the `tiddlywiki.info` file +* Custom plugins are unpacked into their own folder + +A common usage is to convert a TiddlyWiki HTML file into a wiki folder: + +``` +tiddlywiki --load ./mywiki.html --savewikifolder ./mywikifolder +``` diff --git a/core/modules/commands/savewikifolder.js b/core/modules/commands/savewikifolder.js new file mode 100644 index 000000000..d29292742 --- /dev/null +++ b/core/modules/commands/savewikifolder.js @@ -0,0 +1,190 @@ +/*\ +title: $:/core/modules/commands/savewikifolder.js +type: application/javascript +module-type: command + +Command to save the current wiki as a wiki folder + +--savewikifolder [] [] + +\*/ +(function(){ + +/*jslint node: true, browser: true */ +/*global $tw: false */ +"use strict"; + +exports.info = { + name: "savewikifolder", + synchronous: true +}; + +var fs,path; +if($tw.node) { + fs = require("fs"); + path = require("path"); +} + +var Command = function(params,commander,callback) { + this.params = params; + this.commander = commander; + this.callback = callback; +}; + +Command.prototype.execute = function() { + if(this.params.length < 1) { + return "Missing wiki folder path"; + } + var wikifoldermaker = new WikiFolderMaker(this.params[0],this.params[1],this.commander); + return wikifoldermaker.save(); +}; + +function WikiFolderMaker(wikiFolderPath,wikiFilter,commander) { + this.wikiFolderPath = wikiFolderPath; + this.wikiFilter = wikiFilter || "[all[tiddlers]]"; + this.commander = commander; + this.wiki = commander.wiki; + this.savedPaths = []; // So that we can detect filename clashes +} + +WikiFolderMaker.prototype.log = function(str) { + if(this.commander.verbose) { + console.log(str); + } +}; + +WikiFolderMaker.prototype.tiddlersToIgnore = [ + "$:/boot/boot.css", + "$:/boot/boot.js", + "$:/boot/bootprefix.js", + "$:/core", + "$:/library/sjcl.js", + "$:/temp/info-plugin" +]; + +/* +Returns null if successful, or an error string if there was an error +*/ +WikiFolderMaker.prototype.save = function() { + var self = this; + // Check that the output directory doesn't exist + if(fs.existsSync(this.wikiFolderPath) && !$tw.utils.isDirectoryEmpty(this.wikiFolderPath)) { + return "The unpackwiki command requires that the output wiki folder be empty"; + } + // Get the tiddlers from the source wiki + var tiddlerTitles = this.wiki.filterTiddlers(this.wikiFilter); + // Initialise a new tiddlwiki.info file + var newWikiInfo = {}; + // Process each incoming tiddler in turn + $tw.utils.each(tiddlerTitles,function(title) { + var tiddler = self.wiki.getTiddler(title); + if(tiddler) { + if(self.tiddlersToIgnore.indexOf(title) !== -1) { + // Ignore the core plugin and the ephemeral info plugin + self.log("Ignoring tiddler: " + title); + } else { + var type = tiddler.fields.type, + pluginType = tiddler.fields["plugin-type"]; + if(type === "application/json" && pluginType) { + // Plugin tiddler + var libraryDetails = self.findPluginInLibrary(title); + if(libraryDetails) { + // A plugin from the core library + self.log("Adding built-in plugin: " + libraryDetails.name); + newWikiInfo[libraryDetails.type] = newWikiInfo[libraryDetails.type] || []; + $tw.utils.pushTop(newWikiInfo[libraryDetails.type],libraryDetails.name); + } else { + // A custom plugin + self.log("Processing custom plugin: " + title); + self.saveCustomPlugin(tiddler); + } + } else { + // Ordinary tiddler + self.saveTiddler("tiddlers",tiddler); + } + } + } + }); + // Save the tiddlywiki.info file + this.saveJSONFile("tiddlywiki.info",newWikiInfo); + self.log("Writing tiddlywiki.info: " + JSON.stringify(newWikiInfo,null,$tw.config.preferences.jsonSpaces)); + return null; +}; + +/* +Test whether the specified tiddler is a plugin in the plugin library +*/ +WikiFolderMaker.prototype.findPluginInLibrary = function(title) { + var parts = title.split("/"), + pluginPath, type, name; + if(parts[0] === "$:") { + if(parts[1] === "languages" && parts.length === 3) { + pluginPath = "languages" + path.sep + parts[2]; + type = parts[1]; + name = parts[2]; + } else if(parts[1] === "plugins" || parts[1] === "themes" && parts.length === 4) { + pluginPath = parts[1] + path.sep + parts[2] + path.sep + parts[3]; + type = parts[1]; + name = parts[2] + "/" + parts[3]; + } + } + if(pluginPath && type && name) { + pluginPath = path.resolve($tw.boot.bootPath,"..",pluginPath); + if(fs.existsSync(pluginPath)) { + return { + pluginPath: pluginPath, + type: type, + name: name + }; + } + } + return false; +}; + +WikiFolderMaker.prototype.saveCustomPlugin = function(pluginTiddler) { + var self = this, + pluginTitle = pluginTiddler.fields.title, + titleParts = pluginTitle.split("/"), + directory = $tw.utils.generateTiddlerFilepath(titleParts[titleParts.length - 1],{ + directory: path.resolve(this.wikiFolderPath,pluginTiddler.fields["plugin-type"] + "s") + }), + pluginInfo = { + title: pluginTitle, + description: pluginTiddler.fields.description, + author: pluginTiddler.fields.author, + "core-version": pluginTiddler.fields["core-version"], + list: pluginTiddler.fields.list + }; + this.saveJSONFile(directory + path.sep + "plugin.info",pluginInfo); + self.log("Writing " + directory + path.sep + "plugin.info: " + JSON.stringify(pluginInfo,null,$tw.config.preferences.jsonSpaces)); + var pluginTiddlers = JSON.parse(pluginTiddler.fields.text).tiddlers; // A hashmap of tiddlers in the plugin + $tw.utils.each(pluginTiddlers,function(tiddler) { + self.saveTiddler(directory,new $tw.Tiddler(tiddler)); + }); +}; + +WikiFolderMaker.prototype.saveTiddler = function(directory,tiddler) { + var fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{ + directory: path.resolve(this.wikiFolderPath,directory), + wiki: this.wiki + }); + $tw.utils.saveTiddlerToFileSync(tiddler,fileInfo); +}; + +WikiFolderMaker.prototype.saveJSONFile = function(filename,json) { + this.saveTextFile(filename,JSON.stringify(json,null,$tw.config.preferences.jsonSpaces)); +}; + +WikiFolderMaker.prototype.saveTextFile = function(filename,data) { + this.saveFile(filename,"utf8",data); +}; + +WikiFolderMaker.prototype.saveFile = function(filename,encoding,data) { + var filepath = path.resolve(this.wikiFolderPath,filename); + $tw.utils.createFileDirectories(filepath); + fs.writeFileSync(filepath,data,encoding); +}; + +exports.Command = Command; + +})(); diff --git a/core/modules/utils/filesystem.js b/core/modules/utils/filesystem.js index 8d07a9c45..a2011c433 100644 --- a/core/modules/utils/filesystem.js +++ b/core/modules/utils/filesystem.js @@ -222,7 +222,7 @@ exports.generateTiddlerFileInfo = function(tiddler,options) { // Take the file extension from the tiddler content type var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""}; // Generate the filepath - fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{ + fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{ extension: contentTypeInfo.extension, directory: options.directory, pathFilters: options.pathFilters @@ -238,7 +238,7 @@ Options include: pathFilters: optional array of filters to be used to generate the base path wiki: optional wiki for evaluating the pathFilters */ -exports.generateTiddlerFilepath = function(tiddler,options) { +exports.generateTiddlerFilepath = function(title,options) { var self = this, directory = options.directory || "", extension = options.extension || "", @@ -247,7 +247,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) { if(options.pathFilters && options.wiki) { $tw.utils.each(options.pathFilters,function(filter) { if(!filepath) { - var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]), + var source = options.wiki.makeTiddlerIterator([title]), result = options.wiki.filterTiddlers(filter,null,source); if(result.length > 0) { filepath = result[0]; @@ -257,7 +257,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) { } // If not, generate a base pathname if(!filepath) { - filepath = tiddler.fields.title; + filepath = title; // If the filepath already ends in the extension then remove it if(filepath.substring(filepath.length - extension.length) === extension) { filepath = filepath.substring(0,filepath.length - extension.length); @@ -275,7 +275,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) { if(!filepath) { // ...then just use the character codes of the title filepath = ""; - $tw.utils.each(tiddler.fields.title.split(""),function(char) { + $tw.utils.each(title.split(""),function(char) { if(filepath) { filepath += "-"; } @@ -304,18 +304,41 @@ exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) { if(fileInfo.hasMetaFile) { // Save the tiddler as a separate body and meta file var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; - fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) { + fs.writeFile(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding,function(err) { if(err) { return callback(err); } - fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback); + fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),"utf8",callback); }); } else { // Save the tiddler as a self contained templated file if(fileInfo.type === "application/x-tiddler") { - fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback); + fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8",callback); } else { - fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback); + fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),"utf8",callback); + } + } +}; + +/* +Save a tiddler to a file described by the fileInfo: + filepath: the absolute path to the file containing the tiddler + type: the type of the tiddler file (NOT the type of the tiddler) + hasMetaFile: true if the file also has a companion .meta file +*/ +exports.saveTiddlerToFileSync = function(tiddler,fileInfo) { + $tw.utils.createDirectory(path.dirname(fileInfo.filepath)); + if(fileInfo.hasMetaFile) { + // Save the tiddler as a separate body and meta file + var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; + fs.writeFileSync(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding); + fs.writeFileSync(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),"utf8"); + } else { + // Save the tiddler as a self contained templated file + if(fileInfo.type === "application/x-tiddler") { + fs.writeFileSync(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8"); + } else { + fs.writeFileSync(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),"utf8"); } } }; diff --git a/editions/tw5.com/tiddlers/commands/SaveWikiFolderCommand.tid b/editions/tw5.com/tiddlers/commands/SaveWikiFolderCommand.tid new file mode 100644 index 000000000..a079cc21f --- /dev/null +++ b/editions/tw5.com/tiddlers/commands/SaveWikiFolderCommand.tid @@ -0,0 +1,7 @@ +title: SaveWikiFolderCommand +tags: Commands +created: 20190414110120829 +modified: 20190414110120829 +caption: savewikifolder + +{{$:/language/Help/savewikifolder}}