Fix filesystem adaptor (#5113)

* ignore .env

testing new implementation

almost there

closer

bug, desyncing

fixed

final testing

final testing

cleanup

cleanup

* isEditableFile flow fixed

* removed `basepath` logic

* callback to delete title from $tw.boot.files

* comment fix

* have syncer delete from boot.files

* syntax

* bugfix: error on missing directory

* bugifx

* remove !draft check

* fix relative filepaths

* cleanup

* cleanup !draft

* catch undefined filepaths in deleteTiddlerFile()

* typo

* whitelist wiki dir, encodeURIComponent otherwise

* test for wikiPath, not wikiPath/tiddlers

* don't need to .normailze()

* whitelist wiki directory, move cleanup to util

* use cleanup util & fail EPERM & EACCESS gracefully

* comments

* final bugs fixed

* improved sync error

boot/boot.js

@@ -1915,15 +1915,21 @@ $tw.loadTiddlersFromSpecification = function(filepath,excludeRegExp) {
 			}
 		} else {
 			// Process directory specifier
-			var dirPath = path.resolve(filepath,dirSpec.path),
+			var dirPath = path.resolve(filepath,dirSpec.path);
-				files = fs.readdirSync(dirPath),
+			if(fs.existsSync(dirPath) && fs.statSync(dirPath).isDirectory()) {
-				fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
+				var	files = fs.readdirSync(dirPath),
-				metaRegExp = /^.*\.meta$/;
+					fileRegExp = new RegExp(dirSpec.filesRegExp || "^.*$"),
-			for(var t=0; t<files.length; t++) {
+					metaRegExp = /^.*\.meta$/;
-				var filename = files[t];
+				for(var t=0; t<files.length; t++) {
-				if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
+					var filename = files[t];
-					processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
+					if(filename !== "tiddlywiki.files" && !metaRegExp.test(filename) && fileRegExp.test(filename)) {
+						processFile(dirPath + path.sep + filename,dirSpec.isTiddlerFile,dirSpec.fields,dirSpec.isEditableFile);
+					}
 				}
+			} else {
+				console.log("Warning: a directory in a tiddlywiki.files file does not exist.");
+				console.log("dirPath: " + dirPath);	
+				console.log("tiddlywiki.files location: " + filepath);
 			}
 		}
 	});

core/modules/syncer.js

@@ -633,6 +633,10 @@ DeleteTiddlerTask.prototype.run = function(callback) {
 		}
 		// Remove the info stored about this tiddler
 		delete self.syncer.tiddlerInfo[self.title];
+		if($tw.boot.files){
+			// Remove the tiddler from $tw.boot.files
+			delete $tw.boot.files[self.title];
+		}
 		// Invoke the callback
 		callback(null);
 	},{

core/modules/utils/filesystem.js

@@ -204,15 +204,17 @@ exports.deleteEmptyDirs = function(dirpath,callback) {
 /*
 Create a fileInfo object for saving a tiddler:
 	filepath: the absolute path to the file containing the tiddler
-	type: the type of the tiddler file (NOT the type of the tiddler)
+	type: the type of the tiddler file on disk (NOT the type of the tiddler)
 	hasMetaFile: true if the file also has a companion .meta file
 Options include:
 	directory: absolute path of root directory to which we are saving
 	pathFilters: optional array of filters to be used to generate the base path
-	wiki: optional wiki for evaluating the pathFilters
+	extFilters: optional array of filters to be used to generate the base path
+	wiki: optional wiki for evaluating the pathFilters,
+	fileInfo: an existing fileInfo to check against
 */
 exports.generateTiddlerFileInfo = function(tiddler,options) {
-	var fileInfo = {};
+	var fileInfo = {}, metaExt;
 	// Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace
 	var hasUnsafeFields = false;
 	$tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) {
@@ -238,19 +240,66 @@ exports.generateTiddlerFileInfo = function(tiddler,options) {
 			fileInfo.type = tiddlerType;
 			fileInfo.hasMetaFile = true;
 		}
+		if(options.extFilters) {
+			// Check for extension override
+			metaExt = $tw.utils.generateTiddlerExtension(tiddler.fields.title,{
+				extFilters: options.extFilters,
+				wiki: options.wiki
+			});
+			if(metaExt === ".tid") {
+				// Overriding to the .tid extension needs special handling
+				fileInfo.type = "application/x-tiddler";
+				fileInfo.hasMetaFile = false;
+			} else if (metaExt === ".json") {
+				// Overriding to the .json extension needs special handling
+				fileInfo.type = "application/json";
+				fileInfo.hasMetaFile = false;
+			} else if (metaExt) {
+				//If the new type matches a known extention, use that MIME type's encoding
+				var extInfo = $tw.utils.getFileExtensionInfo(metaExt);
+				fileInfo.type = extInfo ? extInfo.type : null;
+				fileInfo.encoding = $tw.utils.getTypeEncoding(metaExt);
+				fileInfo.hasMetaFile = true;
+			}
+		}
 	}
-	// Take the file extension from the tiddler content type
+	// Take the file extension from the tiddler content type or metaExt
 	var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
 	// Generate the filepath
 	fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{
-		extension: contentTypeInfo.extension,
+		extension: metaExt || contentTypeInfo.extension,
 		directory: options.directory,
 		pathFilters: options.pathFilters,
-		wiki: options.wiki
+		wiki: options.wiki,
+		fileInfo: options.fileInfo
 	});
 	return fileInfo;
 };
 
+/*
+Generate the file extension for saving a tiddler
+Options include:
+	extFilters: optional array of filters to be used to generate the extention
+	wiki: optional wiki for evaluating the extFilters
+*/
+exports.generateTiddlerExtension = function(title,options) {
+	var self = this,
+		extension;
+	// Check if any of the extFilters applies
+	if(options.extFilters && options.wiki) { 
+		$tw.utils.each(options.extFilters,function(filter) {
+			if(!extension) {
+				var source = options.wiki.makeTiddlerIterator([title]),
+					result = options.wiki.filterTiddlers(filter,null,source);
+				if(result.length > 0) {
+					extension = result[0];
+				}
+			}
+		});
+	}
+	return extension;
+};
+
 /*
 Generate the filepath for saving a tiddler
 Options include:
@@ -258,12 +307,13 @@ Options include:
 	directory: absolute path of root directory to which we are saving
 	pathFilters: optional array of filters to be used to generate the base path
 	wiki: optional wiki for evaluating the pathFilters
+	fileInfo: an existing fileInfo object to check against
 */
 exports.generateTiddlerFilepath = function(title,options) {
 	var self = this,
 		directory = options.directory || "",
 		extension = options.extension || "",
-		filepath;
+		filepath;	
 	// Check if any of the pathFilters applies
 	if(options.pathFilters && options.wiki) {
 		$tw.utils.each(options.pathFilters,function(filter) {
@@ -276,7 +326,6 @@ exports.generateTiddlerFilepath = function(title,options) {
 			}
 		});
 	}
-	// If not, generate a base pathname
 	if(!filepath) {
 		filepath = title;
 		// If the filepath already ends in the extension then remove it
@@ -286,10 +335,13 @@ exports.generateTiddlerFilepath = function(title,options) {
 		// Remove any forward or backward slashes so we don't create directories
 		filepath = filepath.replace(/\/|\\/g,"_");
 	}
-	// Don't let the filename start with a dot because such files are invisible on *nix
+	//If the path does not start with "." or ".." and a path seperator, then
-	filepath = filepath.replace(/^\./g,"_");
+	if(!/^\.{1,2}[/\\]/g.test(filepath)) {
+		// Don't let the filename start with any dots because such files are invisible on *nix
+		filepath = filepath.replace(/^\.+/g,"_");
+	}
 	// Remove any characters that can't be used in cross-platform filenames
-	filepath = $tw.utils.transliterate(filepath.replace(/<|>|\:|\"|\||\?|\*|\^/g,"_"));
+	filepath = $tw.utils.transliterate(filepath.replace(/<|>|~|\:|\"|\||\?|\*|\^/g,"_"));
 	// Truncate the filename if it is too long
 	if(filepath.length > 200) {
 		filepath = filepath.substr(0,200);
@@ -306,12 +358,21 @@ exports.generateTiddlerFilepath = function(title,options) {
 		});
 	}
 	// Add a uniquifier if the file already exists
-	var fullPath,
+	var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined,
 		count = 0;
 	do {
 		fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension);
+		if(oldPath && oldPath == fullPath) {
+			break;
+		}
 		count++;
 	} while(fs.existsSync(fullPath));
+	//If the path does not start with the wiki directory, or if the last write failed
+	var encode = fullPath.indexOf($tw.boot.wikiPath) !== 0 || ((options.fileInfo || {writeError: false}).writeError == true);
+	if(encode){
+		//encodeURIComponent() and then resolve to tiddler directory
+		fullPath = path.resolve(directory, encodeURIComponent(fullPath));
+	}
 	// Return the full path to the file
 	return fullPath;
 };
@@ -366,4 +427,58 @@ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) {
 	}
 };
 
+/*
+Delete a file described by the fileInfo if it exits
+*/
+exports.deleteTiddlerFile = function(fileInfo, callback) {
+	//Only attempt to delete files that exist on disk
+	if(!fileInfo.filepath || !fs.existsSync(fileInfo.filepath)) {
+		return callback(null);
+	}
+	// Delete the file
+	fs.unlink(fileInfo.filepath,function(err) {
+		if(err) {
+			return callback(err);
+		}	
+		// Delete the metafile if present
+		if(fileInfo.hasMetaFile && fs.existsSync(fileInfo.filepath + ".meta")) {
+			fs.unlink(fileInfo.filepath + ".meta",function(err) {
+				if(err) {
+					return callback(err);
+				}
+				return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
+			});
+		} else {
+			return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
+		}
+	});
+};
+
+/*
+Cleanup old files on disk, by comparing the options values:
+	adaptorInfo from $tw.syncer.tiddlerInfo
+	bootInfo from $tw.boot.files
+*/
+exports.cleanupTiddlerFiles = function(options, callback) {
+	var adaptorInfo = options.adaptorInfo || {},
+	bootInfo = options.bootInfo || {},
+	title = options.title || "undefined";
+	if(adaptorInfo.filepath && bootInfo.filepath && adaptorInfo.filepath !== bootInfo.filepath) {
+		return $tw.utils.deleteTiddlerFile(adaptorInfo, function(err){
+			if(err) {
+				if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
+					// Error deleting the previous file on disk, should fail gracefully
+					$tw.syncer.displayError("Server desynchronized. Error cleaning up previous file for tiddler: "+title, err);
+					return callback(null);
+				} else {
+					return callback(err);
+				}
+			}
+			return callback(null);
+		});
+	} else {
+		return callback(null);
+	}
+};
+
 })();

plugins/tiddlywiki/filesystem/filesystemadaptor.js

@@ -35,7 +35,9 @@ FileSystemAdaptor.prototype.isReady = function() {
 };
 
 FileSystemAdaptor.prototype.getTiddlerInfo = function(tiddler) {
-	return {};
+	//Returns the existing fileInfo for the tiddler. To regenerate, call getTiddlerFileInfo().
+	var title = tiddler.fields.title;
+	return this.boot.files[title];
 };
 
 /*
@@ -44,24 +46,25 @@ Return a fileInfo object for a tiddler, creating it if necessary:
   type: the type of the tiddler file (NOT the type of the tiddler -- see below)
   hasMetaFile: true if the file also has a companion .meta file
 
-The boot process populates this.boot.files for each of the tiddler files that it loads. The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
+The boot process populates this.boot.files for each of the tiddler files that it loads.
+The type is found by looking up the extension in $tw.config.fileExtensionInfo (eg "application/x-tiddler" for ".tid" files).
 
 It is the responsibility of the filesystem adaptor to update this.boot.files for new files that are created.
 */
 FileSystemAdaptor.prototype.getTiddlerFileInfo = function(tiddler,callback) {
 	// See if we've already got information about this file
 	var title = tiddler.fields.title,
-		fileInfo = this.boot.files[title];
+		newInfo, fileInfo = this.boot.files[title];
-	if(!fileInfo) {
+	// Always generate a fileInfo object when this fuction is called
-		// Otherwise, we'll need to generate it
+	newInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
-		fileInfo = $tw.utils.generateTiddlerFileInfo(tiddler,{
+		directory: this.boot.wikiTiddlersPath,
-			directory: this.boot.wikiTiddlersPath,
+		pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
-			pathFilters: this.wiki.getTiddlerText("$:/config/FileSystemPaths","").split("\n"),
+		extFilters: this.wiki.getTiddlerText("$:/config/FileSystemExtensions","").split("\n"),
-			wiki: this.wiki
+		wiki: this.wiki,
-		});
+		fileInfo: fileInfo
-		this.boot.files[title] = fileInfo;
+	});
-	}
+	this.boot.files[title] = newInfo;
-	callback(null,fileInfo);
+	callback(null,newInfo);
 };
 
 
@@ -74,7 +77,31 @@ FileSystemAdaptor.prototype.saveTiddler = function(tiddler,callback) {
 		if(err) {
 			return callback(err);
 		}
-		$tw.utils.saveTiddlerToFile(tiddler,fileInfo,callback);
+		$tw.utils.saveTiddlerToFile(tiddler,fileInfo,function(err) {
+			if(err) {
+				if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "open") {
+					var bootInfo = self.boot.files[tiddler.fields.title];
+					bootInfo.writeError = true;
+					self.boot.files[tiddler.fields.title] = bootInfo;
+					$tw.syncer.displayError("Sync for tiddler [["+tiddler.fields.title+"]] will be retried with encoded filepath", encodeURIComponent(bootInfo.filepath));
+					return callback(err);
+				} else {
+					return callback(err);
+				}
+			}
+			// Cleanup duplicates if the file moved or changed extensions
+			var options = {
+				adaptorInfo: ($tw.syncer.tiddlerInfo[tiddler.fields.title] || {adaptorInfo: {} }).adaptorInfo,
+				bootInfo: self.boot.files[tiddler.fields.title] || {},
+				title: tiddler.fields.title
+			};
+			$tw.utils.cleanupTiddlerFiles(options, function(err){
+				if(err) {
+					return callback(err);
+				}
+				return callback(null, self.boot.files[tiddler.fields.title]);
+			});
+		});
 	});
 };
 
@@ -95,22 +122,17 @@ FileSystemAdaptor.prototype.deleteTiddler = function(title,callback,options) {
 		fileInfo = this.boot.files[title];
 	// Only delete the tiddler if we have writable information for the file
 	if(fileInfo) {
-		// Delete the file
+		$tw.utils.deleteTiddlerFile(fileInfo, function(err){
-		fs.unlink(fileInfo.filepath,function(err) {
 			if(err) {
-				return callback(err);
+				if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") {
-			}
+					// Error deleting the file on disk, should fail gracefully
-			// Delete the metafile if present
+					$tw.syncer.displayError("Server desynchronized. Error deleting file for deleted tiddler: "+title, err);
-			if(fileInfo.hasMetaFile) {
+					return callback(null);
-				fs.unlink(fileInfo.filepath + ".meta",function(err) {
+				} else {
-					if(err) {
+					return callback(err);
-						return callback(err);
+				}
-					}
-					return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
-				});
-			} else {
-				return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),callback);
 			}
+			return callback(null);
 		});
 	} else {
 		callback(null);