mirror of
https://github.com/Jermolene/TiddlyWiki5
synced 2024-11-23 10:07:19 +00:00
Filesystemadaptor: Improve handling of JSON files
Fixes #3875 * Use .json files (instead of .tid) for any tiddler whose fields contain values that can't be stored as a .tid file * Save application/json tiddlers as .json files * Refactor most of the file handling as re-usable utilities
This commit is contained in:
Jermolene
parent
edd3156430
commit
7fcd2f132e
@ -181,4 +181,143 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
|
||||
});
|
||||
};
|
||||
|
||||
/*
|
||||
Create a fileInfo object for saving a tiddler:
|
||||
filepath: the absolute path to the file containing the tiddler
|
||||
type: the type of the tiddler file (NOT the type of the tiddler)
|
||||
hasMetaFile: true if the file also has a companion .meta file
|
||||
Options include:
|
||||
directory: absolute path of root directory to which we are saving
|
||||
pathFilters: optional array of filters to be used to generate the base path
|
||||
wiki: optional wiki for evaluating the pathFilters
|
||||
*/
|
||||
exports.generateTiddlerFileInfo = function(tiddler,options) {
|
||||
var fileInfo = {};
|
||||
// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
|
||||
var hasUnsafeFields = false;
|
||||
$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
|
||||
if(fieldName !== "text") {
|
||||
hasUnsafeFields = hasUnsafeFields || /[\x00-\x1F]/mg.test(value);
|
||||
hasUnsafeFields = hasUnsafeFields || ($tw.utils.trim(value) !== value);
|
||||
}
|
||||
});
|
||||
// Check for field values
|
||||
if(hasUnsafeFields) {
|
||||
// Save as a JSON file
|
||||
fileInfo.type = "application/json";
|
||||
fileInfo.hasMetaFile = false;
|
||||
} else {
|
||||
// Save as a .tid or a text/binary file plus a .meta file
|
||||
var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
|
||||
if(tiddlerType === "text/vnd.tiddlywiki") {
|
||||
// Save as a .tid file
|
||||
fileInfo.type = "application/x-tiddler";
|
||||
fileInfo.hasMetaFile = false;
|
||||
} else {
|
||||
// Save as a text/binary file and a .meta file
|
||||
fileInfo.type = tiddlerType;
|
||||
fileInfo.hasMetaFile = true;
|
||||
}
|
||||
}
|
||||
// Take the file extension from the tiddler content type
|
||||
var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
|
||||
// Generate the filepath
|
||||
fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler,{
|
||||
extension: contentTypeInfo.extension,
|
||||
directory: options.directory,
|
||||
pathFilters: options.pathFilters
|
||||
});
|
||||
return fileInfo;
|
||||
};
|
||||
|
||||
/*
|
||||
Generate the filepath for saving a tiddler
|
||||
Options include:
|
||||
extension: file extension to be added the finished filepath
|
||||
directory: absolute path of root directory to which we are saving
|
||||
pathFilters: optional array of filters to be used to generate the base path
|
||||
wiki: optional wiki for evaluating the pathFilters
|
||||
*/
|
||||
exports.generateTiddlerFilepath = function(tiddler,options) {
|
||||
var self = this,
|
||||
directory = options.directory || "",
|
||||
extension = options.extension || "",
|
||||
filepath;
|
||||
// Check if any of the pathFilters applies
|
||||
if(options.pathFilters && options.wiki) {
|
||||
$tw.utils.each(options.pathFilters,function(filter) {
|
||||
if(!filepath) {
|
||||
var source = options.wiki.makeTiddlerIterator([tiddler.fields.title]),
|
||||
result = options.wiki.filterTiddlers(filter,null,source);
|
||||
if(result.length > 0) {
|
||||
filepath = result[0];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
// If not, generate a base pathname
|
||||
if(!filepath) {
|
||||
filepath = tiddler.fields.title;
|
||||
// If the filepath already ends in the extension then remove it
|
||||
if(filepath.substring(filepath.length - extension.length) === extension) {
|
||||
filepath = filepath.substring(0,filepath.length - extension.length);
|
||||
}
|
||||
// Remove any forward or backward slashes so we don't create directories
|
||||
filepath = filepath.replace(/\/|\\/g,"_");
|
||||
}
|
||||
// Remove any characters that can't be used in cross-platform filenames
|
||||
filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
|
||||
// Truncate the filename if it is too long
|
||||
if(filepath.length > 200) {
|
||||
filepath = filepath.substr(0,200);
|
||||
}
|
||||
// If the resulting filename is blank (eg because the title is just punctuation characters)
|
||||
if(!filepath) {
|
||||
// ...then just use the character codes of the title
|
||||
filepath = "";
|
||||
$tw.utils.each(tiddler.fields.title.split(""),function(char) {
|
||||
if(filepath) {
|
||||
filepath += "-";
|
||||
}
|
||||
filepath += char.charCodeAt(0).toString();
|
||||
});
|
||||
}
|
||||
// Add a uniquifier if the file already exists
|
||||
var fullPath,
|
||||
count = 0;
|
||||
do {
|
||||
fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
|
||||
count++;
|
||||
} while(fs.existsSync(fullPath));
|
||||
// Return the full path to the file
|
||||
return fullPath;
|
||||
};
|
||||
|
||||
/*
|
||||
Save a tiddler to a file described by the fileInfo:
|
||||
filepath: the absolute path to the file containing the tiddler
|
||||
type: the type of the tiddler file (NOT the type of the tiddler)
|
||||
hasMetaFile: true if the file also has a companion .meta file
|
||||
*/
|
||||
exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) {
|
||||
$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
|
||||
if(fileInfo.hasMetaFile) {
|
||||
// Save the tiddler as a separate body and meta file
|
||||
var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
|
||||
fs.writeFile(fileInfo.filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text"]}),{encoding: "utf8"},callback);
|
||||
});
|
||||
} else {
|
||||
// Save the tiddler as a self contained templated file
|
||||
if(fileInfo.type === "application/x-tiddler") {
|
||||
fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),{encoding: "utf8"},callback);
|
||||
} else {
|
||||
fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings()],null,$tw.config.preferences.jsonSpaces),{encoding: "utf8"},callback);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
})();
|
||||
|
||||
@ -47,98 +47,20 @@ It is the responsibility of the filesystem adaptor to update $tw.boot.files for
|
||||
*/
|
||||
FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
|
||||
// See if we've already got information about this file
|
||||
var self = this,
|
||||
title = tiddler.fields.title,
|
||||
var title = tiddler.fields.title,
|
||||
fileInfo = $tw.boot.files[title];
|
||||
if(fileInfo) {
|
||||
// If so, just invoke the callback
|
||||
callback(null,fileInfo);
|
||||
} else {
|
||||
if(!fileInfo) {
|
||||
// Otherwise, we'll need to generate it
|
||||
fileInfo = {};
|
||||
var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
|
||||
// Get the content type info
|
||||
var contentTypeInfo = $tw.config.contentTypeInfo[tiddlerType] || {};
|
||||
// Get the file type by looking up the extension
|
||||
var extension = contentTypeInfo.extension || ".tid";
|
||||
fileInfo.type = ($tw.config.fileExtensionInfo[extension] || {type: "application/x-tiddler"}).type;
|
||||
// Use a .meta file unless we're saving a .tid file.
|
||||
// (We would need more complex logic if we supported other template rendered tiddlers besides .tid)
|
||||
fileInfo.hasMetaFile = (fileInfo.type !== "application/x-tiddler") && (fileInfo.type !== "application/json");
|
||||
if(!fileInfo.hasMetaFile) {
|
||||
extension = ".tid";
|
||||
}
|
||||
// Generate the base filepath and ensure the directories exist
|
||||
var baseFilepath = path.resolve($tw.boot.wikiTiddlersPath,this.generateTiddlerBaseFilepath(title));
|
||||
$tw.utils.createFileDirectories(baseFilepath);
|
||||
// Start by getting a list of the existing files in the directory
|
||||
fs.readdir(path.dirname(baseFilepath),function(err,files) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
// Start with the base filename plus the extension
|
||||
var filepath = baseFilepath;
|
||||
if(filepath.substr(-extension.length).toLocaleLowerCase() !== extension.toLocaleLowerCase()) {
|
||||
filepath = filepath + extension;
|
||||
}
|
||||
var filename = path.basename(filepath),
|
||||
count = 1;
|
||||
// Add a discriminator if we're clashing with an existing filename while
|
||||
// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
|
||||
while(files.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
|
||||
filepath = baseFilepath + " " + (count++) + extension;
|
||||
filename = path.basename(filepath);
|
||||
}
|
||||
// Set the final fileInfo
|
||||
fileInfo.filepath = filepath;
|
||||
console.log("\x1b[1;35m" + "For " + title + ", type is " + fileInfo.type + " hasMetaFile is " + fileInfo.hasMetaFile + " filepath is " + fileInfo.filepath + "\x1b[0m");
|
||||
$tw.boot.files[title] = fileInfo;
|
||||
// Pass it to the callback
|
||||
callback(null,fileInfo);
|
||||
fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
|
||||
directory: $tw.boot.wikiTiddlersPath,
|
||||
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths"),
|
||||
wiki: this.wiki
|
||||
});
|
||||
$tw.boot.files[title] = fileInfo;
|
||||
}
|
||||
callback(null,fileInfo);
|
||||
};
|
||||
|
||||
/*
|
||||
Given a list of filters, apply every one in turn to source, and return the first result of the first filter with non-empty result.
|
||||
*/
|
||||
FileSystemAdaptor.prototype.findFirstFilter = function(filters,source) {
|
||||
for(var i=0; i<filters.length; i++) {
|
||||
var result = this.wiki.filterTiddlers(filters[i],null,source);
|
||||
if(result.length > 0) {
|
||||
return result[0];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
/*
|
||||
Given a tiddler title and an array of existing filenames, generate a new legal filename for the title, case insensitively avoiding the array of existing filenames
|
||||
*/
|
||||
FileSystemAdaptor.prototype.generateTiddlerBaseFilepath = function(title) {
|
||||
var baseFilename;
|
||||
// Check whether the user has configured a tiddler -> pathname mapping
|
||||
var pathNameFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths");
|
||||
if(pathNameFilters) {
|
||||
var source = this.wiki.makeTiddlerIterator([title]);
|
||||
baseFilename = this.findFirstFilter(pathNameFilters.split("\n"),source);
|
||||
if(baseFilename) {
|
||||
// Interpret "/" and "\" as path separator
|
||||
baseFilename = baseFilename.replace(/\/|\\/g,path.sep);
|
||||
}
|
||||
}
|
||||
if(!baseFilename) {
|
||||
// No mappings provided, or failed to match this tiddler so we use title as filename
|
||||
baseFilename = title.replace(/\/|\\/g,"_");
|
||||
}
|
||||
// Remove any of the characters that are illegal in Windows filenames
|
||||
var baseFilename = $tw.utils.transliterate(baseFilename.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
|
||||
// Truncate the filename if it is too long
|
||||
if(baseFilename.length > 200) {
|
||||
baseFilename = baseFilename.substr(0,200);
|
||||
}
|
||||
return baseFilename;
|
||||
};
|
||||
|
||||
/*
|
||||
Save a tiddler and invoke the callback with (err,adaptorInfo,revision)
|
||||
@ -149,38 +71,7 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
var filepath = fileInfo.filepath,
|
||||
error = $tw.utils.createDirectory(path.dirname(filepath));
|
||||
if(error) {
|
||||
return callback(error);
|
||||
}
|
||||
if(fileInfo.hasMetaFile) {
|
||||
// Save the tiddler as a separate body and meta file
|
||||
var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
|
||||
fs.writeFile(filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
content = self.wiki.renderTiddler("text/plain","$:/core/templates/tiddler-metadata",{variables: {currentTiddler: tiddler.fields.title}});
|
||||
fs.writeFile(fileInfo.filepath + ".meta",content,{encoding: "utf8"},function (err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
self.logger.log("Saved file",filepath);
|
||||
return callback(null);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
// Save the tiddler as a self contained templated file
|
||||
var content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
|
||||
fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
self.logger.log("Saved file",filepath);
|
||||
return callback(null);
|
||||
});
|
||||
}
|
||||
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback);
|
||||
});
|
||||
};
|
||||
|
||||
@ -206,7 +97,6 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
self.logger.log("Deleted file",fileInfo.filepath);
|
||||
// Delete the metafile if present
|
||||
if(fileInfo.hasMetaFile) {
|
||||
fs.unlink(fileInfo.filepath + ".meta",function(err) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user