From 7fcd2f132e91aa375a1fece501b14e991ba1bd4c Mon Sep 17 00:00:00 2001 From: Jermolene Date: Sat, 13 Apr 2019 14:59:44 +0100 Subject: [PATCH] Filesystemadaptor: Improve handling of JSON files Fixes #3875 * Use .json files (instead of .tid) for any tiddler whose fields contain values that can't be stored as a .tid file * Save application/json tiddlers as .json files * Refactor most of the file handling as re-usable utilities --- core/modules/utils/filesystem.js | 139 ++++++++++++++++++ .../filesystem/filesystemadaptor.js | 128 ++-------------- 2 files changed, 148 insertions(+), 119 deletions(-) diff --git a/core/modules/utils/filesystem.js b/core/modules/utils/filesystem.js index 57e916108..8d07a9c45 100644 --- a/core/modules/utils/filesystem.js +++ b/core/modules/utils/filesystem.js @@ -181,4 +181,143 @@ exports.deleteEmptyDirs = function(dirpath,callback) { }); }; +/* +Create a fileInfo object for saving a tiddler: + filepath: the absolute path to the file containing the tiddler + type: the type of the tiddler file (NOT the type of the tiddler) + hasMetaFile: true if the file also has a companion .meta file +Options include: + directory: absolute path of root directory to which we are saving + pathFilters: optional array of filters to be used to generate the base path + wiki: optional wiki for evaluating the pathFilters +*/ +exports.generateTiddlerFileInfo = function(tiddler,options) { + var fileInfo = {}; + // Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace + var hasUnsafeFields = false; + $tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) { + if(fieldName !== "text") { + hasUnsafeFields = hasUnsafeFields || /[\x00-\x1F]/mg.test(value); + hasUnsafeFields = hasUnsafeFields || ($tw.utils.trim(value) !== value); + } + }); + // Check for field values + if(hasUnsafeFields) { + // Save as a JSON file + fileInfo.type = "application/json"; + fileInfo.hasMetaFile = false; + } else { + // Save as a .tid or a text/binary file plus a .meta file + var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki"; + if(tiddlerType === "text/vnd.tiddlywiki") { + // Save as a .tid file + fileInfo.type = "application/x-tiddler"; + fileInfo.hasMetaFile = false; + } else { + // Save as a text/binary file and a .meta file + fileInfo.type = tiddlerType; + fileInfo.hasMetaFile = true; + } + } + // Take the file extension from the tiddler content type + var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""}; + // Generate the filepath + fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{ + extension: contentTypeInfo.extension, + directory: options.directory, + pathFilters: options.pathFilters + }); + return fileInfo; +}; + +/* +Generate the filepath for saving a tiddler +Options include: + extension: file extension to be added the finished filepath + directory: absolute path of root directory to which we are saving + pathFilters: optional array of filters to be used to generate the base path + wiki: optional wiki for evaluating the pathFilters +*/ +exports.generateTiddlerFilepath = function(tiddler,options) { + var self = this, + directory = options.directory || "", + extension = options.extension || "", + filepath; + // Check if any of the pathFilters applies + if(options.pathFilters && options.wiki) { + $tw.utils.each(options.pathFilters,function(filter) { + if(!filepath) { + var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]), + result = options.wiki.filterTiddlers(filter,null,source); + if(result.length > 0) { + filepath = result[0]; + } + } + }); + } + // If not, generate a base pathname + if(!filepath) { + filepath = tiddler.fields.title; + // If the filepath already ends in the extension then remove it + if(filepath.substring(filepath.length - extension.length) === extension) { + filepath = filepath.substring(0,filepath.length - extension.length); + } + // Remove any forward or backward slashes so we don't create directories + filepath = filepath.replace(/\/|\\/g,"_"); + } + // Remove any characters that can't be used in cross-platform filenames + filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_")); + // Truncate the filename if it is too long + if(filepath.length > 200) { + filepath = filepath.substr(0,200); + } + // If the resulting filename is blank (eg because the title is just punctuation characters) + if(!filepath) { + // ...then just use the character codes of the title + filepath = ""; + $tw.utils.each(tiddler.fields.title.split(""),function(char) { + if(filepath) { + filepath += "-"; + } + filepath += char.charCodeAt(0).toString(); + }); + } + // Add a uniquifier if the file already exists + var fullPath, + count = 0; + do { + fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension); + count++; + } while(fs.existsSync(fullPath)); + // Return the full path to the file + return fullPath; +}; + +/* +Save a tiddler to a file described by the fileInfo: + filepath: the absolute path to the file containing the tiddler + type: the type of the tiddler file (NOT the type of the tiddler) + hasMetaFile: true if the file also has a companion .meta file +*/ +exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) { + $tw.utils.createDirectory(path.dirname(fileInfo.filepath)); + if(fileInfo.hasMetaFile) { + // Save the tiddler as a separate body and meta file + var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; + fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) { + if(err) { + return callback(err); + } + fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback); + }); + } else { + // Save the tiddler as a self contained templated file + if(fileInfo.type === "application/x-tiddler") { + fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback); + } else { + fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback); + } + } +}; + })(); diff --git a/plugins/tiddlywiki/filesystem/filesystemadaptor.js b/plugins/tiddlywiki/filesystem/filesystemadaptor.js index c824bd0fb..1b4ddf18f 100644 --- a/plugins/tiddlywiki/filesystem/filesystemadaptor.js +++ b/plugins/tiddlywiki/filesystem/filesystemadaptor.js @@ -47,98 +47,20 @@ It is the responsibility of the filesystem adaptor to update $tw.boot.files for */ FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) { // See if we've already got information about this file - var self = this, - title = tiddler.fields.title, + var title = tiddler.fields.title, fileInfo = $tw.boot.files[title]; - if(fileInfo) { - // If so, just invoke the callback - callback(null,fileInfo); - } else { + if(!fileInfo) { // Otherwise, we'll need to generate it - fileInfo = {}; - var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki"; - // Get the content type info - var contentTypeInfo = $tw.config.contentTypeInfo[tiddlerType] || {}; - // Get the file type by looking up the extension - var extension = contentTypeInfo.extension || ".tid"; - fileInfo.type = ($tw.config.fileExtensionInfo[extension] || {type: "application/x-tiddler"}).type; - // Use a .meta file unless we're saving a .tid file. - // (We would need more complex logic if we supported other template rendered tiddlers besides .tid) - fileInfo.hasMetaFile = (fileInfo.type !== "application/x-tiddler") && (fileInfo.type !== "application/json"); - if(!fileInfo.hasMetaFile) { - extension = ".tid"; - } - // Generate the base filepath and ensure the directories exist - var baseFilepath = path.resolve($tw.boot.wikiTiddlersPath,this.generateTiddlerBaseFilepath(title)); - $tw.utils.createFileDirectories(baseFilepath); - // Start by getting a list of the existing files in the directory - fs.readdir(path.dirname(baseFilepath),function(err,files) { - if(err) { - return callback(err); - } - // Start with the base filename plus the extension - var filepath = baseFilepath; - if(filepath.substr(-extension.length).toLocaleLowerCase() !== extension.toLocaleLowerCase()) { - filepath = filepath + extension; - } - var filename = path.basename(filepath), - count = 1; - // Add a discriminator if we're clashing with an existing filename while - // handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.) - while(files.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) { - filepath = baseFilepath + " " + (count++) + extension; - filename = path.basename(filepath); - } - // Set the final fileInfo - fileInfo.filepath = filepath; -console.log("\x1b[1;35m" + "For " + title + ", type is " + fileInfo.type + " hasMetaFile is " + fileInfo.hasMetaFile + " filepath is " + fileInfo.filepath + "\x1b[0m"); - $tw.boot.files[title] = fileInfo; - // Pass it to the callback - callback(null,fileInfo); + fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{ + directory: $tw.boot.wikiTiddlersPath, + pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths"), + wiki: this.wiki }); + $tw.boot.files[title] = fileInfo; } + callback(null,fileInfo); }; -/* -Given a list of filters, apply every one in turn to source, and return the first result of the first filter with non-empty result. -*/ -FileSystemAdaptor.prototype.findFirstFilter = function(filters,source) { - for(var i=0; i 0) { - return result[0]; - } - } - return null; -}; - -/* -Given a tiddler title and an array of existing filenames, generate a new legal filename for the title, case insensitively avoiding the array of existing filenames -*/ -FileSystemAdaptor.prototype.generateTiddlerBaseFilepath = function(title) { - var baseFilename; - // Check whether the user has configured a tiddler -> pathname mapping - var pathNameFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths"); - if(pathNameFilters) { - var source = this.wiki.makeTiddlerIterator([title]); - baseFilename = this.findFirstFilter(pathNameFilters.split("\n"),source); - if(baseFilename) { - // Interpret "/" and "\" as path separator - baseFilename = baseFilename.replace(/\/|\\/g,path.sep); - } - } - if(!baseFilename) { - // No mappings provided, or failed to match this tiddler so we use title as filename - baseFilename = title.replace(/\/|\\/g,"_"); - } - // Remove any of the characters that are illegal in Windows filenames - var baseFilename = $tw.utils.transliterate(baseFilename.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_")); - // Truncate the filename if it is too long - if(baseFilename.length > 200) { - baseFilename = baseFilename.substr(0,200); - } - return baseFilename; -}; /* Save a tiddler and invoke the callback with (err,adaptorInfo,revision) @@ -149,38 +71,7 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) { if(err) { return callback(err); } - var filepath = fileInfo.filepath, - error = $tw.utils.createDirectory(path.dirname(filepath)); - if(error) { - return callback(error); - } - if(fileInfo.hasMetaFile) { - // Save the tiddler as a separate body and meta file - var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; - fs.writeFile(filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) { - if(err) { - return callback(err); - } - content = self.wiki.renderTiddler("text/plain","$:/core/templates/tiddler-metadata",{variables: {currentTiddler: tiddler.fields.title}}); - fs.writeFile(fileInfo.filepath + ".meta",content,{encoding: "utf8"},function (err) { - if(err) { - return callback(err); - } - self.logger.log("Saved file",filepath); - return callback(null); - }); - }); - } else { - // Save the tiddler as a self contained templated file - var content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}}); - fs.writeFile(filepath,content,{encoding: "utf8"},function (err) { - if(err) { - return callback(err); - } - self.logger.log("Saved file",filepath); - return callback(null); - }); - } + $tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback); }); }; @@ -206,7 +97,6 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) { if(err) { return callback(err); } - self.logger.log("Deleted file",fileInfo.filepath); // Delete the metafile if present if(fileInfo.hasMetaFile) { fs.unlink(fileInfo.filepath + ".meta",function(err) {