mirrors/TiddlyWiki5
1
0
Fork 0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-06-24 22:33:16 +00:00
Code Issues Releases Wiki Activity

Major refactoring of filesystemadaptor

Browse Source 
The code here had got a bit broken by some PRs that I should have
checked more carefully. I’ve done a major refactoring which will
hopefully make it easier to understand, and fixes a number of problems:

* Problem with eg .md tiddlers not being deleted correctly
* Problem with Windows path separators not being usable within
$:/config/FileSystemPaths on Windows
* Problem with filename clashes not being detected correctly when
saving to a different directory via $:/config/FileSystemPaths
* Enables slashes within tiddler titles to be mapped into folders
* Enables plain text files like .md and .css to be saved with .meta
files instead of as .tid files (see #2558)
* No longer replaces spaces with underscores

As this is such a major update, I’d be grateful if Node.js users could
give it a careful run through — in particular, you’ll need to try
creating new tiddlers of various types and ensure that the expected
files are created.
This commit is contained in:
Jermolene 2017-02-11 12:56:42 +00:00
parent 1961db6732
commit 3708f6c8e4
3 changed files with 149 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
core/modules/utils/utils.js
View File

@ -697,7 +697,6 @@ exports.tagToCssSelector = function(tagName) {
});
};
/*
IE does not have sign function
*/
@ -725,4 +724,82 @@ exports.strEndsWith = function(str,ending,position) {
}
};
/*
Transliterate string from eg. Cyrillic Russian to Latin
*/
var transliterationPairs = {
"Ё":"YO",
"Й":"I",
"Ц":"TS",
"У":"U",
"К":"K",
"Е":"E",
"Н":"N",
"Г":"G",
"Ш":"SH",
"Щ":"SCH",
"З":"Z",
"Х":"H",
"Ъ":"'",
"ё":"yo",
"й":"i",
"ц":"ts",
"у":"u",
"к":"k",
"е":"e",
"н":"n",
"г":"g",
"ш":"sh",
"щ":"sch",
"з":"z",
"х":"h",
"ъ":"'",
"Ф":"F",
"Ы":"I",
"В":"V",
"А":"a",
"П":"P",
"Р":"R",
"О":"O",
"Л":"L",
"Д":"D",
"Ж":"ZH",
"Э":"E",
"ф":"f",
"ы":"i",
"в":"v",
"а":"a",
"п":"p",
"р":"r",
"о":"o",
"л":"l",
"д":"d",
"ж":"zh",
"э":"e",
"Я":"Ya",
"Ч":"CH",
"С":"S",
"М":"M",
"И":"I",
"Т":"T",
"Ь":"'",
"Б":"B",
"Ю":"YU",
"я":"ya",
"ч":"ch",
"с":"s",
"м":"m",
"и":"i",
"т":"t",
"ь":"'",
"б":"b",
"ю":"yu"
};
exports.transliterate = function(str) {
return str.split("").map(function(char) {
return transliterationPairs[char] || char;
}).join("");
};
})();

6
editions/tw5.com/tiddlers/nodejs/Customising Tiddler File Naming.tid
View File

@ -6,9 +6,9 @@ type: text/vnd.tiddlywiki
By default, a [[TiddlyWiki on Node.js]] instance using a [[wiki folder|TiddlyWikiFolders]] will create new tiddler files by using the sanitised and disambiguated title as filename.
This can be customised by creating a tiddler [[$:/config/FileSystemPaths]] containing one or more [[filter expressions|Filter Syntax]], each on a line of its own. Newly created tiddlers are matched to each filter in turn, and the first output of the first filter to produce any output is taken as a logical path to be used for the tiddler file. Logical paths don't include the `.tid` extension, and they always use `/` as directory separator (when generating the physical path, this is replaced by the correct separator for the platform ~TiddlyWiki is running on). If none of the filters matches, the logical path is simply the title with all occurences of `/` replaced by `_` (for backwards compatibility).
This can be customised by creating a tiddler [[$:/config/FileSystemPaths]] containing one or more [[filter expressions|Filter Syntax]], each on a line of its own. Newly created tiddlers are matched to each filter in turn, and the first output of the first filter to produce any output is taken as a logical path to be used for the tiddler file. Logical paths don't include the `.tid` extension, and they can use `/` or `\` as directory separator (when generating the physical path, this is replaced by the correct separator for the platform ~TiddlyWiki is running on). If none of the filters matches, the logical path is simply the title with all occurences of `/` replaced by `_` (for backwards compatibility).
In both cases, the characters `<>:"\|?*^ ` are replaced by `_` in order to guarantee that the resulting path is legal on all supported platforms.
In both cases, the characters `<>:"\|?*^` are replaced by `_` in order to guarantee that the resulting path is legal on all supported platforms.
!! Example
@ -21,3 +21,5 @@ In both cases, the characters `<>:"\|?*^ ` are replaced by `_` in order to guara
This will store newly created system tiddlers in `tiddlers/_system` (after stripping the `$:/` prefix), tiddlers tagged [[task]] in a subdirectory `tiddlers/mytasks`, and also create subdirectory structures for all other non-draft tiddlers.
Thus, $:/config/FileSystemPaths itself will end up in `tiddlers/_system/config/FileSystemPaths.tid` or `tiddlers\_system\config\FileSystemPaths.tid`, depending on the platform.
The final `[!has[draft.of]]` will match all remaining non-draft tiddlers. Because there was a match, any `/` or `\` in the tiddler title is mapped to a path separator. Thus, `some/thing/entirely/new` will be saved to `tiddlers/some/thing/entirely/new.tid` (ie, the file `new.tid` in a directory called `entirely`).

146
plugins/tiddlywiki/filesystem/filesystemadaptor.js
View File

@ -35,113 +35,107 @@ FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) {
return {};
};
$tw.config.typeInfo = {
"text/vnd.tiddlywiki": {
fileType: "application/x-tiddler",
extension: ".tid"
}
};
/*
Return a fileInfo object for a tiddler, creating it if necessary:
filepath: the absolute path to the file containing the tiddler
type: the type of the tiddler file (NOT the type of the tiddler -- see below)
hasMetaFile: true if the file also has a companion .meta file
The boot process populates $tw.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
It is the responsibility of the filesystem adaptor to update $tw.boot.files for new files that are created.
*/
FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
// See if we've already got information about this file
var self = this,
title = tiddler.fields.title,
fileInfo = $tw.boot.files[title];
// Get information about how to save tiddlers of this type
var type = tiddler.fields.type || "text/vnd.tiddlywiki";
var typeInfo = $tw.config.typeInfo[type] ||
$tw.config.contentTypeInfo[type] ||
$tw.config.typeInfo["text/vnd.tiddlywiki"];
var extension = typeInfo.extension || "";
if(!fileInfo) {
// If not, we'll need to generate it
if(fileInfo) {
// If so, just invoke the callback
callback(null,fileInfo);
} else {
// Otherwise, we'll need to generate it
fileInfo = {};
var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki";
// Get the content type info
var contentTypeInfo = $tw.config.contentTypeInfo[tiddlerType] || {};
// Get the file type by looking up the extension
var extension = contentTypeInfo.extension || ".tid";
fileInfo.type = $tw.config.fileExtensionInfo[extension].type;
// Use a .meta file unless we're saving a .tid file.
// (We would need more complex logic if we supported other template rendered tiddlers besides .tid)
fileInfo.hasMetaFile = (fileInfo.type !== "application/x-tiddler");
// Generate the base filepath and ensure the directories exist
var baseFilepath = path.resolve($tw.boot.wikiTiddlersPath,this.generateTiddlerBaseFilepath(title));
$tw.utils.createDirectory(baseFilepath);
// Start by getting a list of the existing files in the directory
fs.readdir($tw.boot.wikiTiddlersPath,function(err,files) {
fs.readdir(path.dirname(baseFilepath),function(err,files) {
if(err) {
return callback(err);
}
// Assemble the new fileInfo
fileInfo = {};
fileInfo.filepath = $tw.boot.wikiTiddlersPath + path.sep + self.generateTiddlerFilename(title,extension,files);
fileInfo.type = typeInfo.fileType || tiddler.fields.type;
fileInfo.hasMetaFile = typeInfo.hasMetaFile;
// Save the newly created fileInfo
// Start with the base filename plus the extension
var filepath = baseFilepath;
if(filepath.substr(-extension.length).toLocaleLowerCase() !== extension.toLocaleLowerCase()) {
filepath = filepath + extension;
}
var filename = path.basename(filepath),
count = 1;
// Add a discriminator if we're clashing with an existing filename while
// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
while(files.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
filepath = baseFilepath + " " + (count++) + extension;
filename = path.basename(filepath);
}
// Set the final fileInfo
fileInfo.filepath = filepath;
console.log("\x1b[1;35m" + "For " + title + ", type is " + fileInfo.type + " hasMetaFile is " + fileInfo.hasMetaFile + " filepath is " + fileInfo.filepath + "\x1b[0m");
$tw.boot.files[title] = fileInfo;
// Pass it to the callback
callback(null,fileInfo);
});
} else {
// Otherwise just invoke the callback
callback(null,fileInfo);
}
};
/*
Transliterate string from cyrillic russian to latin
*/
var transliterate = function(cyrillyc) {
var a = {"Ё":"YO","Й":"I","Ц":"TS","У":"U","К":"K","Е":"E","Н":"N","Г":"G","Ш":"SH","Щ":"SCH","З":"Z","Х":"H","Ъ":"'","ё":"yo","й":"i","ц":"ts","у":"u","к":"k","е":"e","н":"n","г":"g","ш":"sh","щ":"sch","з":"z","х":"h","ъ":"'","Ф":"F","Ы":"I","В":"V","А":"a","П":"P","Р":"R","О":"O","Л":"L","Д":"D","Ж":"ZH","Э":"E","ф":"f","ы":"i","в":"v","а":"a","п":"p","р":"r","о":"o","л":"l","д":"d","ж":"zh","э":"e","Я":"Ya","Ч":"CH","С":"S","М":"M","И":"I","Т":"T","Ь":"'","Б":"B","Ю":"YU","я":"ya","ч":"ch","с":"s","м":"m","и":"i","т":"t","ь":"'","б":"b","ю":"yu"};
return cyrillyc.split("").map(function (char) {
return a[char] || char;
}).join("");
};
/*
Given a list of filters, apply every one in turn to source, and return the first result of the first filter with non-empty result.
*/
FileSystemAdaptor.prototype.findFirstFilter = function(filters,source) {
var numFilters = filters.length;
for(var i=0; i<numFilters; i++) {
for(var i=0; i<filters.length; i++) {
var result = this.wiki.filterTiddlers(filters[i],null,source);
if(result.length > 0) {
return result[0];
}
}
return null;
};
/*
Add file extension to a file path if it doesn't already exist.
*/
FileSystemAdaptor.addFileExtension = function(file,extension) {
return $tw.utils.strEndsWith(file,extension) ? file : file + extension;
};
/*
Given a tiddler title and an array of existing filenames, generate a new legal filename for the title, case insensitively avoiding the array of existing filenames
*/
FileSystemAdaptor.prototype.generateTiddlerFilename = function(title,extension,existingFilenames) {
FileSystemAdaptor.prototype.generateTiddlerBaseFilepath = function(title) {
var baseFilename;
// Check whether the user has configured a tiddler -> pathname mapping
var pathNameFilters = this.wiki.getTiddlerText("$:/config/FileSystemPaths");
if(pathNameFilters) {
var source = this.wiki.makeTiddlerIterator([title]);
var result = this.findFirstFilter(pathNameFilters.split("\n"),source);
if(result) {
// interpret "/" as path separator
baseFilename = result.replace(/\//g,path.sep);
baseFilename = this.findFirstFilter(pathNameFilters.split("\n"),source);
console.log("baseFilename",baseFilename)
if(baseFilename) {
// Interpret "/" and "\" as path separator
baseFilename = baseFilename.replace(/\/|\\/g,path.sep);
}
}
if(!baseFilename) {
// no mapping configured, or it did not match this tiddler
// in this case, we fall back to legacy behaviour
baseFilename = title.replace(/\//g,"_");
// No mappings provided, or failed to match this tiddler so we use title as filename
baseFilename = title.replace(/\/|\\/g,"_");
}
// Remove any of the characters that are illegal in Windows filenames
var baseFilename = transliterate(baseFilename.replace(/<|>|\:|\"|\\|\||\?|\*|\^|\s/g,"_"));
var baseFilename = $tw.utils.transliterate(baseFilename.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
// Truncate the filename if it is too long
if(baseFilename.length > 200) {
baseFilename = baseFilename.substr(0,200);
}
// Start with the base filename plus the extension
var filename = FileSystemAdaptor.addFileExtension(baseFilename,extension),
count = 1;
// Add a discriminator if we're clashing with an existing filename while
// handling case-insensitive filesystems (NTFS, FAT/FAT32, etc.)
while(existingFilenames.some(function(value) {return value.toLocaleLowerCase() === filename.toLocaleLowerCase();})) {
filename = baseFilename + " " + (count++) + extension;
}
return filename;
return baseFilename;
};
/*
@ -150,45 +144,39 @@ Save a tiddler and invoke the callback with (err,adaptorInfo,revision)
FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
var self = this;
this.getTiddlerFileInfo(tiddler,function(err,fileInfo) {
var template, content, encoding, filepath,
_finish = function() {
callback(null, {}, 0);
};
if(err) {
return callback(err);
}
var error = $tw.utils.createDirectory(path.dirname(fileInfo.filepath));
var filepath = fileInfo.filepath,
error = $tw.utils.createDirectory(path.dirname(filepath));
if(error) {
return callback(error);
}
var typeInfo = $tw.config.contentTypeInfo[fileInfo.type];
if(fileInfo.hasMetaFile || typeInfo.encoding === "base64") {
if(fileInfo.hasMetaFile) {
// Save the tiddler as a separate body and meta file
filepath = fileInfo.filepath;
var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
fs.writeFile(filepath,tiddler.fields.text,{encoding: typeInfo.encoding},function(err) {
if(err) {
return callback(err);
}
content = self.wiki.renderTiddler("text/plain","$:/core/templates/tiddler-metadata",{variables: {currentTiddler: tiddler.fields.title}});
filepath = FileSystemAdaptor.addFileExtension(fileInfo.filepath,".meta");
fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
fs.writeFile(fileInfo.filepath + ".meta",content,{encoding: "utf8"},function (err) {
if(err) {
return callback(err);
}
self.logger.log("Saved file",filepath);
_finish();
return callback(null);
});
});
} else {
// Save the tiddler as a self contained templated file
content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
filepath = FileSystemAdaptor.addFileExtension(fileInfo.filepath,".tid");
var content = self.wiki.renderTiddler("text/plain","$:/core/templates/tid-tiddler",{variables: {currentTiddler: tiddler.fields.title}});
fs.writeFile(filepath,content,{encoding: "utf8"},function (err) {
if(err) {
return callback(err);
}
self.logger.log("Saved file",filepath);
_finish();
return callback(null);
});
}
});
@ -219,14 +207,14 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
self.logger.log("Deleted file",fileInfo.filepath);
// Delete the metafile if present
if(fileInfo.hasMetaFile) {
fs.unlink(FileSystemAdaptor.addFileExtension(fileInfo.filepath,".meta"),function(err) {
fs.unlink(fileInfo.filepath + ".meta",function(err) {
if(err) {
return callback(err);
}
$tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
});
} else {
$tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
}
});
} else {