Filesystemadaptor: Improve handling of JSON files

Fixes #3875

* Use .json files (instead of .tid) for any tiddler whose fields contain values that can't be stored as a .tid file
* Save application/json tiddlers as .json files
* Refactor most of the file handling as re-usable utilities

core/modules/utils/filesystem.js

@@ -181,4 +181,143 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
 	});
 };
 
+/*
+Create a fileInfo object for saving a tiddler:
+	filepath: the absolute path to the file containing the tiddler
+	type: the type of the tiddler file (NOT the type of the tiddler)
+	hasMetaFile: true if the file also has a companion .meta file
+Options include:
+	directory: absolute path of root directory to which we are saving
+	pathFilters: optional array of filters to be used to generate the base path
+	wiki: optional wiki for evaluating the pathFilters
+*/
+exports.generateTiddlerFileInfo = function(tiddler,options) {
+	var fileInfo = {};
+	// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
+	var hasUnsafeFields = false;
+	$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
+		if(fieldName !== "text") {
+			hasUnsafeFields = hasUnsafeFields || /[\x00-\x1F]/mg.test(value);
+			hasUnsafeFields = hasUnsafeFields || ($tw.utils.trim(value) !== value);
+		}
+	});
+	// Check for field values 
+	if(hasUnsafeFields) {
+		// Save as a JSON file
+		fileInfo.type = "application/json";
+		fileInfo.hasMetaFile = false;
+	} else {
+		// Save as a .tid or a text/binary file plus a .meta file
+		var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
+		if(tiddlerType === "text/vnd.tiddlywiki") {
+			// Save as a .tid file
+			fileInfo.type = "application/x-tiddler";
+			fileInfo.hasMetaFile = false;
+		} else {
+			// Save as a text/binary file and a .meta file
+			fileInfo.type = tiddlerType;
+			fileInfo.hasMetaFile = true;
+		}
+	}
+	// Take the file extension from the tiddler content type
+	var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
+	// Generate the filepath
+	fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{
+		extension: contentTypeInfo.extension,
+		directory: options.directory,
+		pathFilters: options.pathFilters
+	});
+	return fileInfo;
+};
+
+/*
+Generate the filepath for saving a tiddler
+Options include:
+	extension: file extension to be added the finished filepath
+	directory: absolute path of root directory to which we are saving
+	pathFilters: optional array of filters to be used to generate the base path
+	wiki: optional wiki for evaluating the pathFilters
+*/
+exports.generateTiddlerFilepath = function(tiddler,options) {
+	var self = this,
+		directory = options.directory || "",
+		extension = options.extension || "",
+		filepath;
+	// Check if any of the pathFilters applies
+	if(options.pathFilters && options.wiki) {
+		$tw.utils.each(options.pathFilters,function(filter) {
+			if(!filepath) {
+				var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]),
+					result = options.wiki.filterTiddlers(filter,null,source);
+				if(result.length > 0) {
+					filepath = result[0];
+				}
+			}
+		});
+	}
+	// If not, generate a base pathname
+	if(!filepath) {
+		filepath = tiddler.fields.title;
+		// If the filepath already ends in the extension then remove it
+		if(filepath.substring(filepath.length - extension.length) === extension) {
+			filepath = filepath.substring(0,filepath.length - extension.length);
+		}
+		// Remove any forward or backward slashes so we don't create directories
+		filepath = filepath.replace(/\/|\\/g,"_");
+	}
+	// Remove any characters that can't be used in cross-platform filenames
+	filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
+	// Truncate the filename if it is too long
+	if(filepath.length > 200) {
+		filepath = filepath.substr(0,200);
+	}
+	// If the resulting filename is blank (eg because the title is just punctuation characters)
+	if(!filepath) {
+		// ...then just use the character codes of the title
+		filepath = "";	
+		$tw.utils.each(tiddler.fields.title.split(""),function(char) {
+			if(filepath) {
+				filepath += "-";
+			}
+			filepath += char.charCodeAt(0).toString();
+		});
+	}
+	// Add a uniquifier if the file already exists
+	var fullPath,
+		count = 0;
+	do {
+		fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
+		count++;
+	} while(fs.existsSync(fullPath));
+	// Return the full path to the file
+	return fullPath;
+};
+
+/*
+Save a tiddler to a file described by the fileInfo:
+	filepath: the absolute path to the file containing the tiddler
+	type: the type of the tiddler file (NOT the type of the tiddler)
+	hasMetaFile: true if the file also has a companion .meta file
+*/
+exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) {
+	$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
+	if(fileInfo.hasMetaFile) {
+		// Save the tiddler as a separate body and meta file
+		var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
+		fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
+			if(err) {
+				return callback(err);
+			}
+			fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback);
+		});
+	} else {
+		// Save the tiddler as a self contained templated file
+		if(fileInfo.type === "application/x-tiddler") {
+			fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback);
+		} else {
+			fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback);
+		}
+	}
+};
+
 })();