Add savewikifolder command

Makes it much easier to convert a TiddlyWiki HTML file into a full wiki folder.

core/language/en-GB/Help/savewikifolder.tid

@@ -0,0 +1,19 @@
+title: $:/language/Help/savewikifolder
+description: Saves a wiki to a new wiki folder
+
+Saves the current wiki as a wiki folder, including tiddlers, plugins and configuration:
+
+```
+--savewikifolder <wikifolderpath> [<filter>]
+```
+
+* The target wiki folder must be empty or non-existent
+* The filter specifies which tiddlers should be included. It is optional, defaulting to `[all[tiddlers]]`
+* Plugins from the official plugin library are replaced with references to those plugins in the `tiddlywiki.info` file
+* Custom plugins are unpacked into their own folder
+
+A common usage is to convert a TiddlyWiki HTML file into a wiki folder:
+
+```
+tiddlywiki --load ./mywiki.html --savewikifolder ./mywikifolder
+```

core/modules/commands/savewikifolder.js

@@ -0,0 +1,190 @@
+/*\
+title: $:/core/modules/commands/savewikifolder.js
+type: application/javascript
+module-type: command
+
+Command to save the current wiki as a wiki folder
+
+--savewikifolder <wikifolderpath> [<filter>] [<options>]
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.info = {
+	name: "savewikifolder",
+	synchronous: true
+};
+
+var fs,path;
+if($tw.node) {
+	fs = require("fs");
+	path = require("path");
+}
+
+var Command = function(params,commander,callback) {
+	this.params = params;
+	this.commander = commander;
+	this.callback = callback;
+};
+
+Command.prototype.execute = function() {
+	if(this.params.length < 1) {
+		return "Missing wiki folder path";
+	}
+	var wikifoldermaker = new WikiFolderMaker(this.params[0],this.params[1],this.commander);
+	return wikifoldermaker.save();
+};
+
+function WikiFolderMaker(wikiFolderPath,wikiFilter,commander) {
+	this.wikiFolderPath = wikiFolderPath;
+	this.wikiFilter = wikiFilter || "[all[tiddlers]]";
+	this.commander = commander;
+	this.wiki = commander.wiki;
+	this.savedPaths = []; // So that we can detect filename clashes
+}
+
+WikiFolderMaker.prototype.log = function(str) {
+	if(this.commander.verbose) {
+		console.log(str);
+	}
+};
+
+WikiFolderMaker.prototype.tiddlersToIgnore = [
+	"$:/boot/boot.css",
+	"$:/boot/boot.js",
+	"$:/boot/bootprefix.js",
+	"$:/core",
+	"$:/library/sjcl.js",
+	"$:/temp/info-plugin"
+];
+
+/*
+Returns null if successful, or an error string if there was an error
+*/
+WikiFolderMaker.prototype.save = function() {
+	var self = this;
+	// Check that the output directory doesn't exist
+	if(fs.existsSync(this.wikiFolderPath) && !$tw.utils.isDirectoryEmpty(this.wikiFolderPath)) {
+		return "The unpackwiki command requires that the output wiki folder be empty";
+	}
+	// Get the tiddlers from the source wiki
+	var tiddlerTitles = this.wiki.filterTiddlers(this.wikiFilter);
+	// Initialise a new tiddlwiki.info file
+	var newWikiInfo = {};
+	// Process each incoming tiddler in turn
+	$tw.utils.each(tiddlerTitles,function(title) {
+		var tiddler = self.wiki.getTiddler(title);
+		if(tiddler) {
+			if(self.tiddlersToIgnore.indexOf(title) !== -1) {
+				// Ignore the core plugin and the ephemeral info plugin
+				self.log("Ignoring tiddler: " + title);
+			} else {
+				var type = tiddler.fields.type,
+					pluginType = tiddler.fields["plugin-type"];
+				if(type === "application/json" && pluginType) {
+					// Plugin tiddler
+					var libraryDetails = self.findPluginInLibrary(title);
+					if(libraryDetails) {
+						// A plugin from the core library
+						self.log("Adding built-in plugin: " + libraryDetails.name);
+						newWikiInfo[libraryDetails.type] = newWikiInfo[libraryDetails.type]  || [];
+						$tw.utils.pushTop(newWikiInfo[libraryDetails.type],libraryDetails.name);
+					} else {
+						// A custom plugin
+						self.log("Processing custom plugin: " + title);
+						self.saveCustomPlugin(tiddler);
+					}				
+				} else {
+					// Ordinary tiddler
+					self.saveTiddler("tiddlers",tiddler);
+				}
+			}
+		}
+	});
+	// Save the tiddlywiki.info file
+	this.saveJSONFile("tiddlywiki.info",newWikiInfo);
+	self.log("Writing tiddlywiki.info: " + JSON.stringify(newWikiInfo,null,$tw.config.preferences.jsonSpaces));
+	return null;
+};
+
+/*
+Test whether the specified tiddler is a plugin in the plugin library
+*/
+WikiFolderMaker.prototype.findPluginInLibrary = function(title) {
+	var parts = title.split("/"),
+		pluginPath, type, name;
+	if(parts[0] === "$:") {
+		if(parts[1] === "languages" && parts.length === 3) {
+			pluginPath = "languages" + path.sep + parts[2];
+			type = parts[1];
+			name = parts[2];
+		} else if(parts[1] === "plugins" || parts[1] === "themes" && parts.length === 4) {
+			pluginPath = parts[1] + path.sep + parts[2] + path.sep + parts[3];
+			type = parts[1];
+			name = parts[2] + "/" + parts[3];
+		}
+	}
+	if(pluginPath && type && name) {
+		pluginPath = path.resolve($tw.boot.bootPath,"..",pluginPath);
+		if(fs.existsSync(pluginPath)) {
+			return {
+				pluginPath: pluginPath,
+				type: type,
+				name: name
+			};
+		}
+	}
+	return false;
+};
+
+WikiFolderMaker.prototype.saveCustomPlugin = function(pluginTiddler) {
+	var self = this,
+		pluginTitle = pluginTiddler.fields.title,
+		titleParts = pluginTitle.split("/"),
+		directory = $tw.utils.generateTiddlerFilepath(titleParts[titleParts.length - 1],{
+			directory: path.resolve(this.wikiFolderPath,pluginTiddler.fields["plugin-type"] + "s")
+		}),
+		pluginInfo = {
+			title: pluginTitle,
+			description: pluginTiddler.fields.description,
+			author: pluginTiddler.fields.author,
+			"core-version": pluginTiddler.fields["core-version"],
+			list: pluginTiddler.fields.list
+		};
+	this.saveJSONFile(directory + path.sep + "plugin.info",pluginInfo);
+	self.log("Writing " + directory + path.sep + "plugin.info: " + JSON.stringify(pluginInfo,null,$tw.config.preferences.jsonSpaces));
+	var pluginTiddlers = JSON.parse(pluginTiddler.fields.text).tiddlers; // A hashmap of tiddlers in the plugin
+	$tw.utils.each(pluginTiddlers,function(tiddler) {
+		self.saveTiddler(directory,new $tw.Tiddler(tiddler));
+	});
+};
+
+WikiFolderMaker.prototype.saveTiddler = function(directory,tiddler) {
+	var fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
+		directory: path.resolve(this.wikiFolderPath,directory),
+		wiki: this.wiki
+	});
+	$tw.utils.saveTiddlerToFileSync(tiddler,fileInfo);
+};
+
+WikiFolderMaker.prototype.saveJSONFile = function(filename,json) {
+	this.saveTextFile(filename,JSON.stringify(json,null,$tw.config.preferences.jsonSpaces));
+};
+
+WikiFolderMaker.prototype.saveTextFile = function(filename,data) {
+	this.saveFile(filename,"utf8",data);
+};
+
+WikiFolderMaker.prototype.saveFile = function(filename,encoding,data) {
+	var filepath = path.resolve(this.wikiFolderPath,filename);
+	$tw.utils.createFileDirectories(filepath);
+	fs.writeFileSync(filepath,data,encoding);
+};
+
+exports.Command = Command;
+
+})();

core/modules/utils/filesystem.js

@@ -222,7 +222,7 @@ exports.generateTiddlerFileInfo = function(tiddler,options) {
 	// Take the file extension from the tiddler content type
 	var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
 	// Generate the filepath
-	fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{
+	fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{
 		extension: contentTypeInfo.extension,
 		directory: options.directory,
 		pathFilters: options.pathFilters
@@ -238,7 +238,7 @@ Options include:
 	pathFilters: optional array of filters to be used to generate the base path
 	wiki: optional wiki for evaluating the pathFilters
 */
-exports.generateTiddlerFilepath = function(tiddler,options) {
+exports.generateTiddlerFilepath = function(title,options) {
 	var self = this,
 		directory = options.directory || "",
 		extension = options.extension || "",
@@ -247,7 +247,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) {
 	if(options.pathFilters && options.wiki) {
 		$tw.utils.each(options.pathFilters,function(filter) {
 			if(!filepath) {
-				var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]),
+				var source = options.wiki.makeTiddlerIterator([title]),
 					result = options.wiki.filterTiddlers(filter,null,source);
 				if(result.length > 0) {
 					filepath = result[0];
@@ -257,7 +257,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) {
 	}
 	// If not, generate a base pathname
 	if(!filepath) {
-		filepath = tiddler.fields.title;
+		filepath = title;
 		// If the filepath already ends in the extension then remove it
 		if(filepath.substring(filepath.length - extension.length) === extension) {
 			filepath = filepath.substring(0,filepath.length - extension.length);
@@ -275,7 +275,7 @@ exports.generateTiddlerFilepath = function(tiddler,options) {
 	if(!filepath) {
 		// ...then just use the character codes of the title
 		filepath = "";	
-		$tw.utils.each(tiddler.fields.title.split(""),function(char) {
+		$tw.utils.each(title.split(""),function(char) {
 			if(filepath) {
 				filepath += "-";
 			}
@@ -304,18 +304,41 @@ exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) {
 	if(fileInfo.hasMetaFile) {
 		// Save the tiddler as a separate body and meta file
 		var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
-		fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
+		fs.writeFile(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding,function(err) {
 			if(err) {
 				return callback(err);
 			}
-			fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback);
+			fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),"utf8",callback);
 		});
 	} else {
 		// Save the tiddler as a self contained templated file
 		if(fileInfo.type === "application/x-tiddler") {
-			fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback);
+			fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8",callback);
 		} else {
-			fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback);
+			fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),"utf8",callback);
+		}
+	}
+};
+
+/*
+Save a tiddler to a file described by the fileInfo:
+	filepath: the absolute path to the file containing the tiddler
+	type: the type of the tiddler file (NOT the type of the tiddler)
+	hasMetaFile: true if the file also has a companion .meta file
+*/
+exports.saveTiddlerToFileSync = function(tiddler,fileInfo) {
+	$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
+	if(fileInfo.hasMetaFile) {
+		// Save the tiddler as a separate body and meta file
+		var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
+		fs.writeFileSync(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding);
+		fs.writeFileSync(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),"utf8");
+	} else {
+		// Save the tiddler as a self contained templated file
+		if(fileInfo.type === "application/x-tiddler") {
+			fs.writeFileSync(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8");
+		} else {
+			fs.writeFileSync(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),"utf8");
 		}
 	}
 };

editions/tw5.com/tiddlers/commands/SaveWikiFolderCommand.tid

@@ -0,0 +1,7 @@
+title: SaveWikiFolderCommand
+tags: Commands
+created: 20190414110120829
+modified: 20190414110120829
+caption: savewikifolder
+
+{{$:/language/Help/savewikifolder}}