Fix filesystem adaptor (#5113)
* ignore .env testing new implementation almost there closer bug, desyncing fixed final testing final testing cleanup cleanup * isEditableFile flow fixed * removed `basepath` logic * callback to delete title from $tw.boot.files * comment fix * have syncer delete from boot.files * syntax * bugfix: error on missing directory * bugifx * remove !draft check * fix relative filepaths * cleanup * cleanup !draft * catch undefined filepaths in deleteTiddlerFile() * typo * whitelist wiki dir, encodeURIComponent otherwise * test for wikiPath, not wikiPath/tiddlers * don't need to .normailze() * whitelist wiki directory, move cleanup to util * use cleanup util & fail EPERM & EACCESS gracefully * comments * final bugs fixed * improved sync error
This commit is contained in:
Joshua Fontany
GitHub
parent
6a319940d3
commit
dde4182830
22
boot/boot.js
22
boot/boot.js
|
|
@ -1915,15 +1915,21 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) {
|
|||
}
|
||||
} else {
|
||||
// Process directory specifier
|
||||
var dirPath = path.resolve(filepath,dirSpec.path),
|
||||
files = fs.readdirSync(dirPath),
|
||||
fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
|
||||
metaRegExp = /^.*\.meta$/;
|
||||
for(var t=0; t<files.length; t++) {
|
||||
var filename = files[t];
|
||||
if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
|
||||
processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
|
||||
var dirPath = path.resolve(filepath,dirSpec.path);
|
||||
if(fs.existsSync(dirPath) && fs.statSync(dirPath).isDirectory()) {
|
||||
var files = fs.readdirSync(dirPath),
|
||||
fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
|
||||
metaRegExp = /^.*\.meta$/;
|
||||
for(var t=0; t<files.length; t++) {
|
||||
var filename = files[t];
|
||||
if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
|
||||
processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log("Warning: a directory in a tiddlywiki.files file does not exist.");
|
||||
console.log("dirPath: " + dirPath);
|
||||
console.log("tiddlywiki.files location: " + filepath);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -633,6 +633,10 @@ DeleteTiddlerTask.prototype.run = function(callback) {
|
|||
}
|
||||
// Remove the info stored about this tiddler
|
||||
delete self.syncer.tiddlerInfo[self.title];
|
||||
if($tw.boot.files){
|
||||
// Remove the tiddler from $tw.boot.files
|
||||
delete $tw.boot.files[self.title];
|
||||
}
|
||||
// Invoke the callback
|
||||
callback(null);
|
||||
},{
|
||||
|
|
|
|||
|
|
@ -204,15 +204,17 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
|
|||
/*
|
||||
Create a fileInfo object for saving a tiddler:
|
||||
filepath: the absolute path to the file containing the tiddler
|
||||
type: the type of the tiddler file (NOT the type of the tiddler)
|
||||
type: the type of the tiddler file on disk (NOT the type of the tiddler)
|
||||
hasMetaFile: true if the file also has a companion .meta file
|
||||
Options include:
|
||||
directory: absolute path of root directory to which we are saving
|
||||
pathFilters: optional array of filters to be used to generate the base path
|
||||
wiki: optional wiki for evaluating the pathFilters
|
||||
extFilters: optional array of filters to be used to generate the base path
|
||||
wiki: optional wiki for evaluating the pathFilters,
|
||||
fileInfo: an existing fileInfo to check against
|
||||
*/
|
||||
exports.generateTiddlerFileInfo = function(tiddler,options) {
|
||||
var fileInfo = {};
|
||||
var fileInfo = {}, metaExt;
|
||||
// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
|
||||
var hasUnsafeFields = false;
|
||||
$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
|
||||
|
|
@ -238,19 +240,66 @@ exports.generateTiddlerFileInfo = function(tiddler,options) {
|
|||
fileInfo.type = tiddlerType;
|
||||
fileInfo.hasMetaFile = true;
|
||||
}
|
||||
if(options.extFilters) {
|
||||
// Check for extension override
|
||||
metaExt = $tw.utils.generateTiddlerExtension(tiddler.fields.title,{
|
||||
extFilters: options.extFilters,
|
||||
wiki: options.wiki
|
||||
});
|
||||
if(metaExt === ".tid") {
|
||||
// Overriding to the .tid extension needs special handling
|
||||
fileInfo.type = "application/x-tiddler";
|
||||
fileInfo.hasMetaFile = false;
|
||||
} else if (metaExt === ".json") {
|
||||
// Overriding to the .json extension needs special handling
|
||||
fileInfo.type = "application/json";
|
||||
fileInfo.hasMetaFile = false;
|
||||
} else if (metaExt) {
|
||||
//If the new type matches a known extention, use that MIME type's encoding
|
||||
var extInfo = $tw.utils.getFileExtensionInfo(metaExt);
|
||||
fileInfo.type = extInfo ? extInfo.type : null;
|
||||
fileInfo.encoding = $tw.utils.getTypeEncoding(metaExt);
|
||||
fileInfo.hasMetaFile = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Take the file extension from the tiddler content type
|
||||
// Take the file extension from the tiddler content type or metaExt
|
||||
var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
|
||||
// Generate the filepath
|
||||
fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{
|
||||
extension: contentTypeInfo.extension,
|
||||
extension: metaExt || contentTypeInfo.extension,
|
||||
directory: options.directory,
|
||||
pathFilters: options.pathFilters,
|
||||
wiki: options.wiki
|
||||
wiki: options.wiki,
|
||||
fileInfo: options.fileInfo
|
||||
});
|
||||
return fileInfo;
|
||||
};
|
||||
|
||||
/*
|
||||
Generate the file extension for saving a tiddler
|
||||
Options include:
|
||||
extFilters: optional array of filters to be used to generate the extention
|
||||
wiki: optional wiki for evaluating the extFilters
|
||||
*/
|
||||
exports.generateTiddlerExtension = function(title,options) {
|
||||
var self = this,
|
||||
extension;
|
||||
// Check if any of the extFilters applies
|
||||
if(options.extFilters && options.wiki) {
|
||||
$tw.utils.each(options.extFilters,function(filter) {
|
||||
if(!extension) {
|
||||
var source = options.wiki.makeTiddlerIterator([title]),
|
||||
result = options.wiki.filterTiddlers(filter,null,source);
|
||||
if(result.length > 0) {
|
||||
extension = result[0];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return extension;
|
||||
};
|
||||
|
||||
/*
|
||||
Generate the filepath for saving a tiddler
|
||||
Options include:
|
||||
|
|
@ -258,12 +307,13 @@ Options include:
|
|||
directory: absolute path of root directory to which we are saving
|
||||
pathFilters: optional array of filters to be used to generate the base path
|
||||
wiki: optional wiki for evaluating the pathFilters
|
||||
fileInfo: an existing fileInfo object to check against
|
||||
*/
|
||||
exports.generateTiddlerFilepath = function(title,options) {
|
||||
var self = this,
|
||||
directory = options.directory || "",
|
||||
extension = options.extension || "",
|
||||
filepath;
|
||||
filepath;
|
||||
// Check if any of the pathFilters applies
|
||||
if(options.pathFilters && options.wiki) {
|
||||
$tw.utils.each(options.pathFilters,function(filter) {
|
||||
|
|
@ -276,7 +326,6 @@ exports.generateTiddlerFilepath = function(title,options) {
|
|||
}
|
||||
});
|
||||
}
|
||||
// If not, generate a base pathname
|
||||
if(!filepath) {
|
||||
filepath = title;
|
||||
// If the filepath already ends in the extension then remove it
|
||||
|
|
@ -286,10 +335,13 @@ exports.generateTiddlerFilepath = function(title,options) {
|
|||
// Remove any forward or backward slashes so we don't create directories
|
||||
filepath = filepath.replace(/\/|\\/g,"_");
|
||||
}
|
||||
// Don't let the filename start with a dot because such files are invisible on *nix
|
||||
filepath = filepath.replace(/^\./g,"_");
|
||||
//If the path does not start with "." or ".." and a path seperator, then
|
||||
if(!/^\.{1,2}[/\\]/g.test(filepath)) {
|
||||
// Don't let the filename start with any dots because such files are invisible on *nix
|
||||
filepath = filepath.replace(/^\.+/g,"_");
|
||||
}
|
||||
// Remove any characters that can't be used in cross-platform filenames
|
||||
filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
|
||||
filepath = $tw.utils.transliterate(filepath.replace(/<|>|~|\:|\"|\||\?|\*|\^/g,"_"));
|
||||
// Truncate the filename if it is too long
|
||||
if(filepath.length > 200) {
|
||||
filepath = filepath.substr(0,200);
|
||||
|
|
@ -306,12 +358,21 @@ exports.generateTiddlerFilepath = function(title,options) {
|
|||
});
|
||||
}
|
||||
// Add a uniquifier if the file already exists
|
||||
var fullPath,
|
||||
var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined,
|
||||
count = 0;
|
||||
do {
|
||||
fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
|
||||
if(oldPath && oldPath == fullPath) {
|
||||
break;
|
||||
}
|
||||
count++;
|
||||
} while(fs.existsSync(fullPath));
|
||||
//If the path does not start with the wiki directory, or if the last write failed
|
||||
var encode = fullPath.indexOf($tw.boot.wikiPath) !== 0 || ((options.fileInfo || {writeError: false}).writeError == true);
|
||||
if(encode){
|
||||
//encodeURIComponent() and then resolve to tiddler directory
|
||||
fullPath = path.resolve(directory, encodeURIComponent(fullPath));
|
||||
}
|
||||
// Return the full path to the file
|
||||
return fullPath;
|
||||
};
|
||||
|
|
@ -366,4 +427,58 @@ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) {
|
|||
}
|
||||
};
|
||||
|
||||
/*
|
||||
Delete a file described by the fileInfo if it exits
|
||||
*/
|
||||
exports.deleteTiddlerFile = function(fileInfo, callback) {
|
||||
//Only attempt to delete files that exist on disk
|
||||
if(!fileInfo.filepath || !fs.existsSync(fileInfo.filepath)) {
|
||||
return callback(null);
|
||||
}
|
||||
// Delete the file
|
||||
fs.unlink(fileInfo.filepath,function(err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
// Delete the metafile if present
|
||||
if(fileInfo.hasMetaFile && fs.existsSync(fileInfo.filepath + ".meta")) {
|
||||
fs.unlink(fileInfo.filepath + ".meta",function(err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
|
||||
});
|
||||
} else {
|
||||
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/*
|
||||
Cleanup old files on disk, by comparing the options values:
|
||||
adaptorInfo from $tw.syncer.tiddlerInfo
|
||||
bootInfo from $tw.boot.files
|
||||
*/
|
||||
exports.cleanupTiddlerFiles = function(options, callback) {
|
||||
var adaptorInfo = options.adaptorInfo || {},
|
||||
bootInfo = options.bootInfo || {},
|
||||
title = options.title || "undefined";
|
||||
if(adaptorInfo.filepath && bootInfo.filepath && adaptorInfo.filepath !== bootInfo.filepath) {
|
||||
return $tw.utils.deleteTiddlerFile(adaptorInfo, function(err){
|
||||
if(err) {
|
||||
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
|
||||
// Error deleting the previous file on disk, should fail gracefully
|
||||
$tw.syncer.displayError("Server desynchronized. Error cleaning up previous file for tiddler: "+title, err);
|
||||
return callback(null);
|
||||
} else {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
return callback(null);
|
||||
});
|
||||
} else {
|
||||
return callback(null);
|
||||
}
|
||||
};
|
||||
|
||||
})();
|
||||
|
|
|
|||
|
|
@ -35,7 +35,9 @@ FileSystemAdaptor.prototype.isReady = function() {
|
|||
};
|
||||
|
||||
FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) {
|
||||
return {};
|
||||
//Returns the existing fileInfo for the tiddler. To regenerate, call getTiddlerFileInfo().
|
||||
var title = tiddler.fields.title;
|
||||
return this.boot.files[title];
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
@ -44,24 +46,25 @@ Return a fileInfo object for a tiddler, creating it if necessary:
|
|||
type: the type of the tiddler file (NOT the type of the tiddler -- see below)
|
||||
hasMetaFile: true if the file also has a companion .meta file
|
||||
|
||||
The boot process populates this.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
|
||||
The boot process populates this.boot.files for each of the tiddler files that it loads.
|
||||
The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
|
||||
|
||||
It is the responsibility of the filesystem adaptor to update this.boot.files for new files that are created.
|
||||
*/
|
||||
FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
|
||||
// See if we've already got information about this file
|
||||
var title = tiddler.fields.title,
|
||||
fileInfo = this.boot.files[title];
|
||||
if(!fileInfo) {
|
||||
// Otherwise, we'll need to generate it
|
||||
fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
|
||||
directory: this.boot.wikiTiddlersPath,
|
||||
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
|
||||
wiki: this.wiki
|
||||
});
|
||||
this.boot.files[title] = fileInfo;
|
||||
}
|
||||
callback(null,fileInfo);
|
||||
newInfo, fileInfo = this.boot.files[title];
|
||||
// Always generate a fileInfo object when this fuction is called
|
||||
newInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
|
||||
directory: this.boot.wikiTiddlersPath,
|
||||
pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
|
||||
extFilters: this.wiki.getTiddlerText("$:/config/FileSystemExtensions","").split("\n"),
|
||||
wiki: this.wiki,
|
||||
fileInfo: fileInfo
|
||||
});
|
||||
this.boot.files[title] = newInfo;
|
||||
callback(null,newInfo);
|
||||
};
|
||||
|
||||
|
||||
|
|
@ -74,7 +77,31 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
|
|||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback);
|
||||
$tw.utils.saveTiddlerToFile(tiddler,fileInfo,function(err) {
|
||||
if(err) {
|
||||
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "open") {
|
||||
var bootInfo = self.boot.files[tiddler.fields.title];
|
||||
bootInfo.writeError = true;
|
||||
self.boot.files[tiddler.fields.title] = bootInfo;
|
||||
$tw.syncer.displayError("Sync for tiddler [["+tiddler.fields.title+"]] will be retried with encoded filepath", encodeURIComponent(bootInfo.filepath));
|
||||
return callback(err);
|
||||
} else {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
// Cleanup duplicates if the file moved or changed extensions
|
||||
var options = {
|
||||
adaptorInfo: ($tw.syncer.tiddlerInfo[tiddler.fields.title] || {adaptorInfo: {} }).adaptorInfo,
|
||||
bootInfo: self.boot.files[tiddler.fields.title] || {},
|
||||
title: tiddler.fields.title
|
||||
};
|
||||
$tw.utils.cleanupTiddlerFiles(options, function(err){
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, self.boot.files[tiddler.fields.title]);
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -95,22 +122,17 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
|
|||
fileInfo = this.boot.files[title];
|
||||
// Only delete the tiddler if we have writable information for the file
|
||||
if(fileInfo) {
|
||||
// Delete the file
|
||||
fs.unlink(fileInfo.filepath,function(err) {
|
||||
$tw.utils.deleteTiddlerFile(fileInfo, function(err){
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
// Delete the metafile if present
|
||||
if(fileInfo.hasMetaFile) {
|
||||
fs.unlink(fileInfo.filepath + ".meta",function(err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
|
||||
});
|
||||
} else {
|
||||
return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
|
||||
if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
|
||||
// Error deleting the file on disk, should fail gracefully
|
||||
$tw.syncer.displayError("Server desynchronized. Error deleting file for deleted tiddler: "+title, err);
|
||||
return callback(null);
|
||||
} else {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
return callback(null);
|
||||
});
|
||||
} else {
|
||||
callback(null);
|
||||
|
|
|
|||
Loading…
Reference in New Issue