mirrors/TiddlyWiki5
1
0
Fork 0
mirror of https://github.com/Jermolene/TiddlyWiki5 synced 2024-11-26 19:47:20 +00:00
Code Issues Releases Wiki Activity

Fix filesystem adaptor (#5113)

Browse Source 
* ignore .env

testing new implementation

almost there

closer

bug, desyncing

fixed

final testing

final testing

cleanup

cleanup

* isEditableFile flow fixed

* removed `basepath` logic

* callback to delete title from $tw.boot.files

* comment fix

* have syncer delete from boot.files

* syntax

* bugfix: error on missing directory

* bugifx

* remove !draft check

* fix relative filepaths

* cleanup

* cleanup !draft

* catch undefined filepaths in deleteTiddlerFile()

* typo

* whitelist wiki dir, encodeURIComponent otherwise

* test for wikiPath, not wikiPath/tiddlers

* don't need to .normailze()

* whitelist wiki directory, move cleanup to util

* use cleanup util & fail EPERM & EACCESS gracefully

* comments

* final bugs fixed

* improved sync error
This commit is contained in:
Joshua Fontany 2020-11-30 14:31:48 -08:00 committed by GitHub
parent 6a319940d3
commit dde4182830
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 195 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
boot/boot.js
View File

@ -1915,15 +1915,21 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) {
}
} else {
// Process directory specifier
var dirPath = path.resolve(filepath,dirSpec.path),
files = fs.readdirSync(dirPath),
fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
metaRegExp = /^.*\.meta$/;
for(var t=0; t<files.length; t++) {
var filename = files[t];
if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
var dirPath = path.resolve(filepath,dirSpec.path);
if(fs.existsSync(dirPath) && fs.statSync(dirPath).isDirectory()) {
var files = fs.readdirSync(dirPath),
fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
metaRegExp = /^.*\.meta$/;
for(var t=0; t<files.length; t++) {
var filename = files[t];
if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
}
}
} else {
console.log("Warning: a directory in a tiddlywiki.files file does not exist.");
console.log("dirPath: " + dirPath);
console.log("tiddlywiki.files location: " + filepath);
}
}
});

4
core/modules/syncer.js
View File

@ -633,6 +633,10 @@ DeleteTiddlerTask.prototype.run = function(callback) {
}
// Remove the info stored about this tiddler
delete self.syncer.tiddlerInfo[self.title];
if($tw.boot.files){
// Remove the tiddler from $tw.boot.files
delete $tw.boot.files[self.title];
}
// Invoke the callback
callback(null);
},{

139
core/modules/utils/filesystem.js
View File

@ -204,15 +204,17 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
/*
Create a fileInfo object for saving a tiddler:
filepath: the absolute path to the file containing the tiddler
type: the type of the tiddler file (NOT the type of the tiddler)
type: the type of the tiddler file on disk (NOT the type of the tiddler)
hasMetaFile: true if the file also has a companion .meta file
Options include:
directory: absolute path of root directory to which we are saving
pathFilters: optional array of filters to be used to generate the base path
wiki: optional wiki for evaluating the pathFilters
extFilters: optional array of filters to be used to generate the base path
wiki: optional wiki for evaluating the pathFilters,
fileInfo: an existing fileInfo to check against
*/
exports.generateTiddlerFileInfo = function(tiddler,options) {
var fileInfo = {};
var fileInfo = {}, metaExt;
// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
var hasUnsafeFields = false;
$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
@ -238,19 +240,66 @@ exports.generateTiddlerFileInfo = function(tiddler,options) {
fileInfo.type = tiddlerType;
fileInfo.hasMetaFile = true;
}
if(options.extFilters) {
// Check for extension override
metaExt = $tw.utils.generateTiddlerExtension(tiddler.fields.title,{
extFilters: options.extFilters,
wiki: options.wiki
});
if(metaExt === ".tid") {
// Overriding to the .tid extension needs special handling
fileInfo.type = "application/x-tiddler";
fileInfo.hasMetaFile = false;
} else if (metaExt === ".json") {
// Overriding to the .json extension needs special handling
fileInfo.type = "application/json";
fileInfo.hasMetaFile = false;
} else if (metaExt) {
//If the new type matches a known extention, use that MIME type's encoding
var extInfo = $tw.utils.getFileExtensionInfo(metaExt);
fileInfo.type = extInfo ? extInfo.type : null;
fileInfo.encoding = $tw.utils.getTypeEncoding(metaExt);
fileInfo.hasMetaFile = true;
}
}
}
// Take the file extension from the tiddler content type
// Take the file extension from the tiddler content type or metaExt
var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
// Generate the filepath
fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{
extension: contentTypeInfo.extension,
extension: metaExt || contentTypeInfo.extension,
directory: options.directory,
pathFilters: options.pathFilters,
wiki: options.wiki
wiki: options.wiki,
fileInfo: options.fileInfo
});
return fileInfo;
};
/*
Generate the file extension for saving a tiddler
Options include:
extFilters: optional array of filters to be used to generate the extention
wiki: optional wiki for evaluating the extFilters
*/
exports.generateTiddlerExtension = function(title,options) {
var self = this,
extension;
// Check if any of the extFilters applies
if(options.extFilters && options.wiki) {
$tw.utils.each(options.extFilters,function(filter) {
if(!extension) {
var source = options.wiki.makeTiddlerIterator([title]),
result = options.wiki.filterTiddlers(filter,null,source);
if(result.length > 0) {
extension = result[0];
}
}
});
}
return extension;
};
/*
Generate the filepath for saving a tiddler
Options include:
@ -258,12 +307,13 @@ Options include:
directory: absolute path of root directory to which we are saving
pathFilters: optional array of filters to be used to generate the base path
wiki: optional wiki for evaluating the pathFilters
fileInfo: an existing fileInfo object to check against
*/
exports.generateTiddlerFilepath = function(title,options) {
var self = this,
directory = options.directory || "",
extension = options.extension || "",
filepath;
filepath;
// Check if any of the pathFilters applies
if(options.pathFilters && options.wiki) {
$tw.utils.each(options.pathFilters,function(filter) {
@ -276,7 +326,6 @@ exports.generateTiddlerFilepath = function(title,options) {
}
});
}
// If not, generate a base pathname
if(!filepath) {
filepath = title;
// If the filepath already ends in the extension then remove it
@ -286,10 +335,13 @@ exports.generateTiddlerFilepath = function(title,options) {
// Remove any forward or backward slashes so we don't create directories
filepath = filepath.replace(/\/|\\/g,"_");
}
// Don't let the filename start with a dot because such files are invisible on *nix
filepath = filepath.replace(/^\./g,"_");
//If the path does not start with "." or ".." and a path seperator, then
if(!/^\.{1,2}[/\\]/g.test(filepath)) {
// Don't let the filename start with any dots because such files are invisible on *nix
filepath = filepath.replace(/^\.+/g,"_");
}
// Remove any characters that can't be used in cross-platform filenames
filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
filepath = $tw.utils.transliterate(filepath.replace(/<|>|~|\:|\"|\||\?|\*|\^/g,"_"));
// Truncate the filename if it is too long
if(filepath.length > 200) {
filepath = filepath.substr(0,200);
@ -306,12 +358,21 @@ exports.generateTiddlerFilepath = function(title,options) {
});
}
// Add a uniquifier if the file already exists
var fullPath,
var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined,
count = 0;
do {
fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
if(oldPath && oldPath == fullPath) {
break;
}
count++;
} while(fs.existsSync(fullPath));
//If the path does not start with the wiki directory, or if the last write failed
var encode = fullPath.indexOf($tw.boot.wikiPath) !== 0 || ((options.fileInfo || {writeError: false}).writeError == true);
if(encode){
//encodeURIComponent() and then resolve to tiddler directory
fullPath = path.resolve(directory, encodeURIComponent(fullPath));
}
// Return the full path to the file
return fullPath;
};
@ -366,4 +427,58 @@ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) {
}
};
/*
Delete a file described by the fileInfo if it exits
*/
exports.deleteTiddlerFile = function(fileInfo, callback) {
//Only attempt to delete files that exist on disk
if(!fileInfo.filepath || !fs.existsSync(fileInfo.filepath)) {
return callback(null);
}
// Delete the file
fs.unlink(fileInfo.filepath,function(err) {
if(err) {
return callback(err);
}
// Delete the metafile if present
if(fileInfo.hasMetaFile && fs.existsSync(fileInfo.filepath + ".meta")) {
fs.unlink(fileInfo.filepath + ".meta",function(err) {
if(err) {
return callback(err);
}
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
});
} else {
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
}
});
};
/*
Cleanup old files on disk, by comparing the options values:
adaptorInfo from $tw.syncer.tiddlerInfo
bootInfo from $tw.boot.files
*/
exports.cleanupTiddlerFiles = function(options, callback) {
var adaptorInfo = options.adaptorInfo || {},
bootInfo = options.bootInfo || {},
title = options.title || "undefined";
if(adaptorInfo.filepath && bootInfo.filepath && adaptorInfo.filepath !== bootInfo.filepath) {
return $tw.utils.deleteTiddlerFile(adaptorInfo, function(err){
if(err) {
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
// Error deleting the previous file on disk, should fail gracefully
$tw.syncer.displayError("Server desynchronized. Error cleaning up previous file for tiddler: "+title, err);
return callback(null);
} else {
return callback(err);
}
}
return callback(null);
});
} else {
return callback(null);
}
};
})();

78
plugins/tiddlywiki/filesystem/filesystemadaptor.js
View File

@ -35,7 +35,9 @@ FileSystemAdaptor.prototype.isReady = function() {
};
FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) {
return {};
//Returns the existing fileInfo for the tiddler. To regenerate, call getTiddlerFileInfo().
var title = tiddler.fields.title;
return this.boot.files[title];
};
/*
@ -44,24 +46,25 @@ Return a fileInfo object for a tiddler, creating it if necessary:
type: the type of the tiddler file (NOT the type of the tiddler -- see below)
hasMetaFile: true if the file also has a companion .meta file
The boot process populates this.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
The boot process populates this.boot.files for each of the tiddler files that it loads.
The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
It is the responsibility of the filesystem adaptor to update this.boot.files for new files that are created.
*/
FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
// See if we've already got information about this file
var title = tiddler.fields.title,
fileInfo = this.boot.files[title];
if(!fileInfo) {
// Otherwise, we'll need to generate it
fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
directory: this.boot.wikiTiddlersPath,
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
wiki: this.wiki
});
this.boot.files[title] = fileInfo;
}
callback(null,fileInfo);
newInfo, fileInfo = this.boot.files[title];
// Always generate a fileInfo object when this fuction is called
newInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
directory: this.boot.wikiTiddlersPath,
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
extFilters: this.wiki.getTiddlerText("$:/config/FileSystemExtensions","").split("\n"),
wiki: this.wiki,
fileInfo: fileInfo
});
this.boot.files[title] = newInfo;
callback(null,newInfo);
};
@ -74,7 +77,31 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
if(err) {
return callback(err);
}
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback);
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,function(err) {
if(err) {
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "open") {
var bootInfo = self.boot.files[tiddler.fields.title];
bootInfo.writeError = true;
self.boot.files[tiddler.fields.title] = bootInfo;
$tw.syncer.displayError("Sync for tiddler [["+tiddler.fields.title+"]] will be retried with encoded filepath", encodeURIComponent(bootInfo.filepath));
return callback(err);
} else {
return callback(err);
}
}
// Cleanup duplicates if the file moved or changed extensions
var options = {
adaptorInfo: ($tw.syncer.tiddlerInfo[tiddler.fields.title] || {adaptorInfo: {} }).adaptorInfo,
bootInfo: self.boot.files[tiddler.fields.title] || {},
title: tiddler.fields.title
};
$tw.utils.cleanupTiddlerFiles(options, function(err){
if(err) {
return callback(err);
}
return callback(null, self.boot.files[tiddler.fields.title]);
});
});
});
};
@ -95,22 +122,17 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
fileInfo = this.boot.files[title];
// Only delete the tiddler if we have writable information for the file
if(fileInfo) {
// Delete the file
fs.unlink(fileInfo.filepath,function(err) {
$tw.utils.deleteTiddlerFile(fileInfo, function(err){
if(err) {
return callback(err);
}
// Delete the metafile if present
if(fileInfo.hasMetaFile) {
fs.unlink(fileInfo.filepath + ".meta",function(err) {
if(err) {
return callback(err);
}
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
});
} else {
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
// Error deleting the file on disk, should fail gracefully
$tw.syncer.displayError("Server desynchronized. Error deleting file for deleted tiddler: "+title, err);
return callback(null);
} else {
return callback(err);
}
}
return callback(null);
});
} else {
callback(null);