1
0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-06-16 02:19:55 +00:00

Filesystemadaptor: Improve handling of JSON files

Fixes #3875

* Use .json files (instead of .tid) for any tiddler whose fields contain values that can't be stored as a .tid file
* Save application/json tiddlers as .json files
* Refactor most of the file handling as re-usable utilities
This commit is contained in:
Jermolene 2019-04-13 14:59:44 +01:00
parent edd3156430
commit 7fcd2f132e
2 changed files with 148 additions and 119 deletions

View File

@ -181,4 +181,143 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
});
};
/*
Create a fileInfo object for saving a tiddler:
filepath: the absolute path to the file containing the tiddler
type: the type of the tiddler file (NOT the type of the tiddler)
hasMetaFile: true if the file also has a companion .meta file
Options include:
directory: absolute path of root directory to which we are saving
pathFilters: optional array of filters to be used to generate the base path
wiki: optional wiki for evaluating the pathFilters
*/
exports.generateTiddlerFileInfo = function(tiddler,options) {
var fileInfo = {};
// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
var hasUnsafeFields = false;
$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
if(fieldName !== "text") {
hasUnsafeFields = hasUnsafeFields || /[\x00-\x1F]/mg.test(value);
hasUnsafeFields = hasUnsafeFields || ($tw.utils.trim(value) !== value);
}
});
// Check for field values
if(hasUnsafeFields) {
// Save as a JSON file
fileInfo.type = "application/json";
fileInfo.hasMetaFile = false;
} else {
// Save as a .tid or a text/binary file plus a .meta file
var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
if(tiddlerType === "text/vnd.tiddlywiki") {
// Save as a .tid file
fileInfo.type = "application/x-tiddler";
fileInfo.hasMetaFile = false;
} else {
// Save as a text/binary file and a .meta file
fileInfo.type = tiddlerType;
fileInfo.hasMetaFile = true;
}
}
// Take the file extension from the tiddler content type
var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
// Generate the filepath
fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{
extension: contentTypeInfo.extension,
directory: options.directory,
pathFilters: options.pathFilters
});
return fileInfo;
};
/*
Generate the filepath for saving a tiddler
Options include:
extension: file extension to be added the finished filepath
directory: absolute path of root directory to which we are saving
pathFilters: optional array of filters to be used to generate the base path
wiki: optional wiki for evaluating the pathFilters
*/
exports.generateTiddlerFilepath = function(tiddler,options) {
var self = this,
directory = options.directory || "",
extension = options.extension || "",
filepath;
// Check if any of the pathFilters applies
if(options.pathFilters && options.wiki) {
$tw.utils.each(options.pathFilters,function(filter) {
if(!filepath) {
var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]),
result = options.wiki.filterTiddlers(filter,null,source);
if(result.length > 0) {
filepath = result[0];
}
}
});
}
// If not, generate a base pathname
if(!filepath) {
filepath = tiddler.fields.title;
// If the filepath already ends in the extension then remove it
if(filepath.substring(filepath.length - extension.length) === extension) {
filepath = filepath.substring(0,filepath.length - extension.length);
}
// Remove any forward or backward slashes so we don't create directories
filepath = filepath.replace(/\/|\\/g,"_");
}
// Remove any characters that can't be used in cross-platform filenames
filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
// Truncate the filename if it is too long
if(filepath.length > 200) {
filepath = filepath.substr(0,200);
}
// If the resulting filename is blank (eg because the title is just punctuation characters)
if(!filepath) {
// ...then just use the character codes of the title
filepath = "";
$tw.utils.each(tiddler.fields.title.split(""),function(char) {
if(filepath) {
filepath += "-";
}
filepath += char.charCodeAt(0).toString();
});
}
// Add a uniquifier if the file already exists
var fullPath,
count = 0;
do {
fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
count++;
} while(fs.existsSync(fullPath));
// Return the full path to the file
return fullPath;
};
/*
Save a tiddler to a file described by the fileInfo:
filepath: the absolute path to the file containing the tiddler
type: the type of the tiddler file (NOT the type of the tiddler)
hasMetaFile: true if the file also has a companion .meta file
*/
exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) {
$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
if(fileInfo.hasMetaFile) {
// Save the tiddler as a separate body and meta file
var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
if(err) {
return callback(err);
}
fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback);
});
} else {
// Save the tiddler as a self contained templated file
if(fileInfo.type === "application/x-tiddler") {
fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback);
} else {
fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback);
}
}
};
})();

View File

@ -47,98 +47,20 @@ It is the responsibility of the filesystem adaptor to update $tw.boot.files for
*/
FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
// See if we've already got information about this file
var self = this,
title = tiddler.fields.title,
var title = tiddler.fields.title,
fileInfo = $tw.boot.files[title];
if(fileInfo) {
// If so, just invoke the callback
callback(null,fileInfo);
} else {
if(!fileInfo) {
// Otherwise, we'll need to generate it
fileInfo = {};
var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
// Get the content type info
var contentTypeInfo = $tw.config.contentTypeInfo[tiddlerType] || {};
// Get the file type by looking up the extension
var extension = contentTypeInfo.extension || ".tid";
fileInfo.type = ($tw.config.fileExtensionInfo[extension] || {type: "application/x-tiddler"}).type;
// Use a .meta file unless we're saving a .tid file.
// (We would need more complex logic if we supported other template rendered tiddlers besides .tid)
fileInfo.hasMetaFile = (fileInfo.type !== "application/x-tiddler") && (fileInfo.type !== "application/json");
if(!fileInfo.hasMetaFile) {
extension = ".tid";
}
// Generate the base filepath and ensure the directories exist
var baseFilepath = path.resolve($tw.boot.wikiTiddlersPath,this.generateTiddlerBaseFilepath(title));
$tw.utils.createFileDirectories(baseFilepath);
// Start by getting a list of the existing files in the directory
fs.readdir(path.dirname(baseFilepath),function(err,files) {
if(err) {
return callback(err);
}
// Start with the base filename plus the extension
var filepath = baseFilepath;
if(filepath.substr(-extension.length).toLocaleLowerCase() !== extension.toLocaleLowerCase()) {
filepath = filepath + extension;
}
var filename = path.basename(filepath),
count = 1;
// Add a discriminator if we're clashing with an existing filename while
// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
while(files.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
filepath = baseFilepath + " " + (count++) + extension;
filename = path.basename(filepath);
}
// Set the final fileInfo
fileInfo.filepath = filepath;
console.log("\x1b[1;35m" + "For " + title + ", type is " + fileInfo.type + " hasMetaFile is " + fileInfo.hasMetaFile + " filepath is " + fileInfo.filepath + "\x1b[0m");
$tw.boot.files[title] = fileInfo;
// Pass it to the callback
callback(null,fileInfo);
fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
directory: $tw.boot.wikiTiddlersPath,
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths"),
wiki: this.wiki
});
$tw.boot.files[title] = fileInfo;
}
callback(null,fileInfo);
};
/*
Given a list of filters, apply every one in turn to source, and return the first result of the first filter with non-empty result.
*/
FileSystemAdaptor.prototype.findFirstFilter = function(filters,source) {
for(var i=0; i<filters.length; i++) {
var result = this.wiki.filterTiddlers(filters[i],null,source);
if(result.length > 0) {
return result[0];
}
}
return null;
};
/*
Given a tiddler title and an array of existing filenames, generate a new legal filename for the title, case insensitively avoiding the array of existing filenames
*/
FileSystemAdaptor.prototype.generateTiddlerBaseFilepath = function(title) {
var baseFilename;
// Check whether the user has configured a tiddler -> pathname mapping
var pathNameFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths");
if(pathNameFilters) {
var source = this.wiki.makeTiddlerIterator([title]);
baseFilename = this.findFirstFilter(pathNameFilters.split("\n"),source);
if(baseFilename) {
// Interpret "/" and "\" as path separator
baseFilename = baseFilename.replace(/\/|\\/g,path.sep);
}
}
if(!baseFilename) {
// No mappings provided, or failed to match this tiddler so we use title as filename
baseFilename = title.replace(/\/|\\/g,"_");
}
// Remove any of the characters that are illegal in Windows filenames
var baseFilename = $tw.utils.transliterate(baseFilename.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
// Truncate the filename if it is too long
if(baseFilename.length > 200) {
baseFilename = baseFilename.substr(0,200);
}
return baseFilename;
};
/*
Save a tiddler and invoke the callback with (err,adaptorInfo,revision)
@ -149,38 +71,7 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
if(err) {
return callback(err);
}
var filepath = fileInfo.filepath,
error = $tw.utils.createDirectory(path.dirname(filepath));
if(error) {
return callback(error);
}
if(fileInfo.hasMetaFile) {
// Save the tiddler as a separate body and meta file
var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
fs.writeFile(filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
if(err) {
return callback(err);
}
content = self.wiki.renderTiddler("text/plain","$:/core/templates/tiddler-metadata",{variables: {currentTiddler: tiddler.fields.title}});
fs.writeFile(fileInfo.filepath + ".meta",content,{encoding: "utf8"},function (err) {
if(err) {
return callback(err);
}
self.logger.log("Saved file",filepath);
return callback(null);
});
});
} else {
// Save the tiddler as a self contained templated file
var content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
if(err) {
return callback(err);
}
self.logger.log("Saved file",filepath);
return callback(null);
});
}
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback);
});
};
@ -206,7 +97,6 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
if(err) {
return callback(err);
}
self.logger.log("Deleted file",fileInfo.filepath);
// Delete the metafile if present
if(fileInfo.hasMetaFile) {
fs.unlink(fileInfo.filepath + ".meta",function(err) {