From dde4182830f11bfdbcca558065f432556ccf4e65 Mon Sep 17 00:00:00 2001 From: Joshua Fontany Date: Mon, 30 Nov 2020 14:31:48 -0800 Subject: [PATCH] Fix filesystem adaptor (#5113) * ignore .env testing new implementation almost there closer bug, desyncing fixed final testing final testing cleanup cleanup * isEditableFile flow fixed * removed `basepath` logic * callback to delete title from $tw.boot.files * comment fix * have syncer delete from boot.files * syntax * bugfix: error on missing directory * bugifx * remove !draft check * fix relative filepaths * cleanup * cleanup !draft * catch undefined filepaths in deleteTiddlerFile() * typo * whitelist wiki dir, encodeURIComponent otherwise * test for wikiPath, not wikiPath/tiddlers * don't need to .normailze() * whitelist wiki directory, move cleanup to util * use cleanup util & fail EPERM & EACCESS gracefully * comments * final bugs fixed * improved sync error --- boot/boot.js | 22 ++- core/modules/syncer.js | 4 + core/modules/utils/filesystem.js | 139 ++++++++++++++++-- .../filesystem/filesystemadaptor.js | 78 ++++++---- 4 files changed, 195 insertions(+), 48 deletions(-) diff --git a/boot/boot.js b/boot/boot.js index 7f85417a6..97bf73837 100644 --- a/boot/boot.js +++ b/boot/boot.js @@ -1915,15 +1915,21 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) { } } else { // Process directory specifier - var dirPath = path.resolve(filepath,dirSpec.path), - files = fs.readdirSync(dirPath), - fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"), - metaRegExp = /^.*\.meta$/; - for(var t=0; t 0) { + extension = result[0]; + } + } + }); + } + return extension; +}; + /* Generate the filepath for saving a tiddler Options include: @@ -258,12 +307,13 @@ Options include: directory: absolute path of root directory to which we are saving pathFilters: optional array of filters to be used to generate the base path wiki: optional wiki for evaluating the pathFilters + fileInfo: an existing fileInfo object to check against */ exports.generateTiddlerFilepath = function(title,options) { var self = this, directory = options.directory || "", extension = options.extension || "", - filepath; + filepath; // Check if any of the pathFilters applies if(options.pathFilters && options.wiki) { $tw.utils.each(options.pathFilters,function(filter) { @@ -276,7 +326,6 @@ exports.generateTiddlerFilepath = function(title,options) { } }); } - // If not, generate a base pathname if(!filepath) { filepath = title; // If the filepath already ends in the extension then remove it @@ -286,10 +335,13 @@ exports.generateTiddlerFilepath = function(title,options) { // Remove any forward or backward slashes so we don't create directories filepath = filepath.replace(/\/|\\/g,"_"); } - // Don't let the filename start with a dot because such files are invisible on *nix - filepath = filepath.replace(/^\./g,"_"); + //If the path does not start with "." or ".." and a path seperator, then + if(!/^\.{1,2}[/\\]/g.test(filepath)) { + // Don't let the filename start with any dots because such files are invisible on *nix + filepath = filepath.replace(/^\.+/g,"_"); + } // Remove any characters that can't be used in cross-platform filenames - filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_")); + filepath = $tw.utils.transliterate(filepath.replace(/<|>|~|\:|\"|\||\?|\*|\^/g,"_")); // Truncate the filename if it is too long if(filepath.length > 200) { filepath = filepath.substr(0,200); @@ -306,12 +358,21 @@ exports.generateTiddlerFilepath = function(title,options) { }); } // Add a uniquifier if the file already exists - var fullPath, + var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined, count = 0; do { fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension); + if(oldPath && oldPath == fullPath) { + break; + } count++; } while(fs.existsSync(fullPath)); + //If the path does not start with the wiki directory, or if the last write failed + var encode = fullPath.indexOf($tw.boot.wikiPath) !== 0 || ((options.fileInfo || {writeError: false}).writeError == true); + if(encode){ + //encodeURIComponent() and then resolve to tiddler directory + fullPath = path.resolve(directory, encodeURIComponent(fullPath)); + } // Return the full path to the file return fullPath; }; @@ -366,4 +427,58 @@ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) { } }; +/* +Delete a file described by the fileInfo if it exits +*/ +exports.deleteTiddlerFile = function(fileInfo, callback) { + //Only attempt to delete files that exist on disk + if(!fileInfo.filepath || !fs.existsSync(fileInfo.filepath)) { + return callback(null); + } + // Delete the file + fs.unlink(fileInfo.filepath,function(err) { + if(err) { + return callback(err); + } + // Delete the metafile if present + if(fileInfo.hasMetaFile && fs.existsSync(fileInfo.filepath + ".meta")) { + fs.unlink(fileInfo.filepath + ".meta",function(err) { + if(err) { + return callback(err); + } + return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback); + }); + } else { + return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback); + } + }); +}; + +/* +Cleanup old files on disk, by comparing the options values: + adaptorInfo from $tw.syncer.tiddlerInfo + bootInfo from $tw.boot.files +*/ +exports.cleanupTiddlerFiles = function(options, callback) { + var adaptorInfo = options.adaptorInfo || {}, + bootInfo = options.bootInfo || {}, + title = options.title || "undefined"; + if(adaptorInfo.filepath && bootInfo.filepath && adaptorInfo.filepath !== bootInfo.filepath) { + return $tw.utils.deleteTiddlerFile(adaptorInfo, function(err){ + if(err) { + if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") { + // Error deleting the previous file on disk, should fail gracefully + $tw.syncer.displayError("Server desynchronized. Error cleaning up previous file for tiddler: "+title, err); + return callback(null); + } else { + return callback(err); + } + } + return callback(null); + }); + } else { + return callback(null); + } +}; + })(); diff --git a/plugins/tiddlywiki/filesystem/filesystemadaptor.js b/plugins/tiddlywiki/filesystem/filesystemadaptor.js index 84da6236e..fce2eaeaa 100644 --- a/plugins/tiddlywiki/filesystem/filesystemadaptor.js +++ b/plugins/tiddlywiki/filesystem/filesystemadaptor.js @@ -35,7 +35,9 @@ FileSystemAdaptor.prototype.isReady = function() { }; FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) { - return {}; + //Returns the existing fileInfo for the tiddler. To regenerate, call getTiddlerFileInfo(). + var title = tiddler.fields.title; + return this.boot.files[title]; }; /* @@ -44,24 +46,25 @@ Return a fileInfo object for a tiddler, creating it if necessary: type: the type of the tiddler file (NOT the type of the tiddler -- see below) hasMetaFile: true if the file also has a companion .meta file -The boot process populates this.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files). +The boot process populates this.boot.files for each of the tiddler files that it loads. +The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files). It is the responsibility of the filesystem adaptor to update this.boot.files for new files that are created. */ FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) { // See if we've already got information about this file var title = tiddler.fields.title, - fileInfo = this.boot.files[title]; - if(!fileInfo) { - // Otherwise, we'll need to generate it - fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{ - directory: this.boot.wikiTiddlersPath, - pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"), - wiki: this.wiki - }); - this.boot.files[title] = fileInfo; - } - callback(null,fileInfo); + newInfo, fileInfo = this.boot.files[title]; + // Always generate a fileInfo object when this fuction is called + newInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{ + directory: this.boot.wikiTiddlersPath, + pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"), + extFilters: this.wiki.getTiddlerText("$:/config/FileSystemExtensions","").split("\n"), + wiki: this.wiki, + fileInfo: fileInfo + }); + this.boot.files[title] = newInfo; + callback(null,newInfo); }; @@ -74,7 +77,31 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) { if(err) { return callback(err); } - $tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback); + $tw.utils.saveTiddlerToFile(tiddler,fileInfo,function(err) { + if(err) { + if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "open") { + var bootInfo = self.boot.files[tiddler.fields.title]; + bootInfo.writeError = true; + self.boot.files[tiddler.fields.title] = bootInfo; + $tw.syncer.displayError("Sync for tiddler [["+tiddler.fields.title+"]] will be retried with encoded filepath", encodeURIComponent(bootInfo.filepath)); + return callback(err); + } else { + return callback(err); + } + } + // Cleanup duplicates if the file moved or changed extensions + var options = { + adaptorInfo: ($tw.syncer.tiddlerInfo[tiddler.fields.title] || {adaptorInfo: {} }).adaptorInfo, + bootInfo: self.boot.files[tiddler.fields.title] || {}, + title: tiddler.fields.title + }; + $tw.utils.cleanupTiddlerFiles(options, function(err){ + if(err) { + return callback(err); + } + return callback(null, self.boot.files[tiddler.fields.title]); + }); + }); }); }; @@ -95,22 +122,17 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) { fileInfo = this.boot.files[title]; // Only delete the tiddler if we have writable information for the file if(fileInfo) { - // Delete the file - fs.unlink(fileInfo.filepath,function(err) { + $tw.utils.deleteTiddlerFile(fileInfo, function(err){ if(err) { - return callback(err); - } - // Delete the metafile if present - if(fileInfo.hasMetaFile) { - fs.unlink(fileInfo.filepath + ".meta",function(err) { - if(err) { - return callback(err); - } - return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback); - }); - } else { - return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback); + if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") { + // Error deleting the file on disk, should fail gracefully + $tw.syncer.displayError("Server desynchronized. Error deleting file for deleted tiddler: "+title, err); + return callback(null); + } else { + return callback(err); + } } + return callback(null); }); } else { callback(null);