/*\ title: $:/core/modules/utils/filesystem.js type: application/javascript module-type: utils-node File system utilities \*/ (function(){ /*jslint node: true, browser: true */ /*global $tw: false */ "use strict"; var fs = require("fs"), path = require("path"); /* Return the subdirectories of a path */ exports.getSubdirectories = function(dirPath) { if(!$tw.utils.isDirectory(dirPath)) { return null; } var subdirs = []; $tw.utils.each(fs.readdirSync(dirPath),function(item) { if($tw.utils.isDirectory(path.resolve(dirPath,item))) { subdirs.push(item); } }); return subdirs; } /* Recursively (and synchronously) copy a directory and all its content */ exports.copyDirectory = function(srcPath,dstPath) { // Remove any trailing path separators srcPath = path.resolve($tw.utils.removeTrailingSeparator(srcPath)); dstPath = path.resolve($tw.utils.removeTrailingSeparator(dstPath)); // Check that neither director is within the other if(srcPath.substring(0,dstPath.length) === dstPath || dstPath.substring(0,srcPath.length) === srcPath) { return "Cannot copy nested directories"; } // Create the destination directory var err = $tw.utils.createDirectory(dstPath); if(err) { return err; } // Function to copy a folder full of files var copy = function(srcPath,dstPath) { var srcStats = fs.lstatSync(srcPath), dstExists = fs.existsSync(dstPath); if(srcStats.isFile()) { $tw.utils.copyFile(srcPath,dstPath); } else if(srcStats.isDirectory()) { var items = fs.readdirSync(srcPath); for(var t=0; t 0) { bytesRead = fs.readSync(srcFile,fileBuffer,0,FILE_BUFFER_LENGTH,pos); fs.writeSync(dstFile,fileBuffer,0,bytesRead); pos += bytesRead; } fs.closeSync(srcFile); fs.closeSync(dstFile); return null; }; /* Remove trailing path separator */ exports.removeTrailingSeparator = function(dirPath) { var len = dirPath.length; if(dirPath.charAt(len-1) === path.sep) { dirPath = dirPath.substr(0,len-1); } return dirPath; }; /* Recursively create a directory */ exports.createDirectory = function(dirPath) { if(dirPath.substr(dirPath.length-1,1) !== path.sep) { dirPath = dirPath + path.sep; } var pos = 1; pos = dirPath.indexOf(path.sep,pos); while(pos !== -1) { var subDirPath = dirPath.substr(0,pos); if(!$tw.utils.isDirectory(subDirPath)) { try { fs.mkdirSync(subDirPath); } catch(e) { return "Error creating directory '" + subDirPath + "'"; } } pos = dirPath.indexOf(path.sep,pos + 1); } return null; }; /* Recursively create directories needed to contain a specified file */ exports.createFileDirectories = function(filePath) { return $tw.utils.createDirectory(path.dirname(filePath)); }; /* Recursively delete a directory */ exports.deleteDirectory = function(dirPath) { if(fs.existsSync(dirPath)) { var entries = fs.readdirSync(dirPath); for(var entryIndex=0; entryIndex 0) { return callback(null); } fs.rmdir(dirpath,function(err) { if(err) { return callback(err); } self.deleteEmptyDirs(path.dirname(dirpath),callback); }); }); }; /* Create a fileInfo object for saving a tiddler: filepath: the absolute path to the file containing the tiddler type: the type of the tiddler file on disk (NOT the type of the tiddler) hasMetaFile: true if the file also has a companion .meta file isEditableFile: true if the tiddler was loaded via non-standard options & marked editable Options include: directory: absolute path of root directory to which we are saving pathFilters: optional array of filters to be used to generate the base path extFilters: optional array of filters to be used to generate the base path wiki: optional wiki for evaluating the pathFilters, fileInfo: an existing fileInfo to check against */ exports.generateTiddlerFileInfo = function(tiddler,options) { var fileInfo = {}, metaExt; // Propagate the isEditableFile flag if(options.fileInfo && !!options.fileInfo.isEditableFile) { fileInfo.isEditableFile = true; fileInfo.originalpath = options.fileInfo.originalpath; } // Check if the tiddler has any unsafe fields that can't be expressed in a .tid or .meta file: containing control characters, or leading/trailing whitespace var hasUnsafeFields = false; $tw.utils.each(tiddler.getFieldStrings(),function(value,fieldName) { if(fieldName !== "text") { hasUnsafeFields = hasUnsafeFields || /[\x00-\x1F]/mg.test(value); hasUnsafeFields = hasUnsafeFields || ($tw.utils.trim(value) !== value); } hasUnsafeFields = hasUnsafeFields || /:/mg.test(fieldName); }); // Check for field values if(hasUnsafeFields) { // Save as a JSON file fileInfo.type = "application/json"; fileInfo.hasMetaFile = false; } else { // Save as a .tid or a text/binary file plus a .meta file var tiddlerType = tiddler.fields.type || "text/vnd.tiddlywiki"; if(tiddlerType === "text/vnd.tiddlywiki") { // Save as a .tid file fileInfo.type = "application/x-tiddler"; fileInfo.hasMetaFile = false; } else { // Save as a text/binary file and a .meta file fileInfo.type = tiddlerType; fileInfo.hasMetaFile = true; } if(options.extFilters) { // Check for extension overrides metaExt = $tw.utils.generateTiddlerExtension(tiddler.fields.title,{ extFilters: options.extFilters, wiki: options.wiki }); if(metaExt) { if(metaExt === ".tid") { // Overriding to the .tid extension needs special handling fileInfo.type = "application/x-tiddler"; fileInfo.hasMetaFile = false; } else if (metaExt === ".json") { // Overriding to the .json extension needs special handling fileInfo.type = "application/json"; fileInfo.hasMetaFile = false; } else { //If the new type matches a known extention, use that MIME type's encoding var extInfo = $tw.utils.getFileExtensionInfo(metaExt); fileInfo.type = extInfo ? extInfo.type : null; fileInfo.encoding = $tw.utils.getTypeEncoding(metaExt); fileInfo.hasMetaFile = true; } } } } // Take the file extension from the tiddler content type or metaExt var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""}; // Generate the filepath fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{ extension: metaExt || contentTypeInfo.extension, directory: options.directory, pathFilters: options.pathFilters, wiki: options.wiki, fileInfo: options.fileInfo }); return fileInfo; }; /* Generate the file extension for saving a tiddler Options include: extFilters: optional array of filters to be used to generate the extention wiki: optional wiki for evaluating the extFilters */ exports.generateTiddlerExtension = function(title,options) { var extension; // Check if any of the extFilters applies if(options.extFilters && options.wiki) { $tw.utils.each(options.extFilters,function(filter) { if(!extension) { var source = options.wiki.makeTiddlerIterator([title]), result = options.wiki.filterTiddlers(filter,null,source); if(result.length > 0) { extension = result[0]; } } }); } return extension; }; /* Generate the filepath for saving a tiddler Options include: extension: file extension to be added the finished filepath directory: absolute path of root directory to which we are saving pathFilters: optional array of filters to be used to generate the base path wiki: optional wiki for evaluating the pathFilters fileInfo: an existing fileInfo object to check against */ exports.generateTiddlerFilepath = function(title,options) { var directory = options.directory || "", extension = options.extension || "", originalpath = (options.fileInfo && options.fileInfo.originalpath) ? options.fileInfo.originalpath : "", filepath; // Check if any of the pathFilters applies if(options.pathFilters && options.wiki) { $tw.utils.each(options.pathFilters,function(filter) { if(!filepath) { var source = options.wiki.makeTiddlerIterator([title]), result = options.wiki.filterTiddlers(filter,null,source); if(result.length > 0) { filepath = result[0]; } } }); } if(!filepath && !!originalpath) { //Use the originalpath without the extension var ext = path.extname(originalpath); filepath = originalpath.substring(0,originalpath.length - ext.length); } else if(!filepath) { filepath = title; // Remove any forward or backward slashes so we don't create directories filepath = filepath.replace(/\/|\\/g,"_"); } // Replace any Windows control codes filepath = filepath.replace(/^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/i,"_$1_"); // Replace any leading spaces with the same number of underscores filepath = filepath.replace(/^ +/,function (u) { return u.replace(/ /g, "_")}); //If the path does not start with "." or ".." && a path seperator, then if(!/^\.{1,2}[/\\]/g.test(filepath)) { // Don't let the filename start with any dots because such files are invisible on *nix filepath = filepath.replace(/^\.+/g,function (u) { return u.replace(/\./g, "_")}); } // Replace any Unicode control codes filepath = filepath.replace(/[\x00-\x1f\x80-\x9f]/g,"_"); // Replace any characters that can't be used in cross-platform filenames filepath = $tw.utils.transliterate(filepath.replace(/<|>|~|\:|\"|\||\?|\*|\^/g,"_")); // Replace any dots or spaces at the end of the extension with the same number of underscores extension = extension.replace(/[\. ]+$/, function (u) { return u.replace(/[\. ]/g, "_")}); // Truncate the extension if it is too long if(extension.length > 32) { extension = extension.substr(0,32); } // If the filepath already ends in the extension then remove it if(filepath.substring(filepath.length - extension.length) === extension) { filepath = filepath.substring(0,filepath.length - extension.length); } // Truncate the filename if it is too long if(filepath.length > 200) { filepath = filepath.substr(0,200); } // If the resulting filename is blank (eg because the title is just punctuation) if(!filepath || /^_+$/g.test(filepath)) { // ...then just use the character codes of the title filepath = ""; $tw.utils.each(title.split(""),function(char) { if(filepath) { filepath += "-"; } filepath += char.charCodeAt(0).toString(); }); } // Add a uniquifier if the file already exists var fullPath, oldPath = (options.fileInfo) ? options.fileInfo.filepath : undefined, count = 0; do { fullPath = path.resolve(directory,filepath + (count ? "_" + count : "") + extension); if(oldPath && oldPath == fullPath) { break; } count++; } while(fs.existsSync(fullPath)); // If the last write failed with an error, or if path does not start with: // the resolved options.directory, the resolved wikiPath directory, the wikiTiddlersPath directory, // or the 'originalpath' directory, then $tw.utils.encodeURIComponentExtended() and resolve to tiddler directory. var writePath = $tw.hooks.invokeHook("th-make-tiddler-path",fullPath,fullPath), encode = (options.fileInfo || {writeError: false}).writeError == true; if(!encode) { encode = !(writePath.indexOf($tw.boot.wikiTiddlersPath) == 0 || writePath.indexOf(path.resolve(directory)) == 0 || writePath.indexOf(path.resolve($tw.boot.wikiPath)) == 0 || writePath.indexOf(path.resolve($tw.boot.wikiTiddlersPath,originalpath)) == 0 ); } if(encode) { writePath = path.resolve(directory,$tw.utils.encodeURIComponentExtended(fullPath)); } // Return the full path to the file return writePath; }; /* Save a tiddler to a file described by the fileInfo: filepath: the absolute path to the file containing the tiddler type: the type of the tiddler file (NOT the type of the tiddler) hasMetaFile: true if the file also has a companion .meta file */ exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) { $tw.utils.createDirectory(path.dirname(fileInfo.filepath)); if(fileInfo.hasMetaFile) { // Save the tiddler as a separate body and meta file var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; fs.writeFile(fileInfo.filepath,tiddler.fields.text || "",typeInfo.encoding,function(err) { if(err) { return callback(err); } fs.writeFile(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text","bag"]}),"utf8",function(err) { if(err) { return callback(err); } return callback(null,fileInfo); }); }); } else { // Save the tiddler as a self contained templated file if(fileInfo.type === "application/x-tiddler") { fs.writeFile(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text","bag"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8",function(err) { if(err) { return callback(err); } return callback(null,fileInfo); }); } else { fs.writeFile(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings({exclude: ["bag"]})],null,$tw.config.preferences.jsonSpaces),"utf8",function(err) { if(err) { return callback(err); } return callback(null,fileInfo); }); } } }; /* Save a tiddler to a file described by the fileInfo: filepath: the absolute path to the file containing the tiddler type: the type of the tiddler file (NOT the type of the tiddler) hasMetaFile: true if the file also has a companion .meta file */ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) { $tw.utils.createDirectory(path.dirname(fileInfo.filepath)); if(fileInfo.hasMetaFile) { // Save the tiddler as a separate body and meta file var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"}; fs.writeFileSync(fileInfo.filepath,tiddler.fields.text || "",typeInfo.encoding); fs.writeFileSync(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text","bag"]}),"utf8"); } else { // Save the tiddler as a self contained templated file if(fileInfo.type === "application/x-tiddler") { fs.writeFileSync(fileInfo.filepath,tiddler.getFieldStringBlock({exclude: ["text","bag"]}) + (!!tiddler.fields.text ? "\n\n" + tiddler.fields.text : ""),"utf8"); } else { fs.writeFileSync(fileInfo.filepath,JSON.stringify([tiddler.getFieldStrings({exclude: ["bag"]})],null,$tw.config.preferences.jsonSpaces),"utf8"); } } return fileInfo; }; /* Delete a file described by the fileInfo if it exits */ exports.deleteTiddlerFile = function(fileInfo,callback) { //Only attempt to delete files that exist on disk if(!fileInfo.filepath || !fs.existsSync(fileInfo.filepath)) { //For some reason, the tiddler is only in memory or we can't modify the file at this path $tw.syncer.displayError("Server deleteTiddlerFile task failed for filepath: "+fileInfo.filepath); return callback(null,fileInfo); } // Delete the file fs.unlink(fileInfo.filepath,function(err) { if(err) { return callback(err); } // Delete the metafile if present if(fileInfo.hasMetaFile && fs.existsSync(fileInfo.filepath + ".meta")) { fs.unlink(fileInfo.filepath + ".meta",function(err) { if(err) { return callback(err); } return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),function(err) { if(err) { return callback(err); } return callback(null,fileInfo); }); }); } else { return $tw.utils.deleteEmptyDirs(path.dirname(fileInfo.filepath),function(err) { if(err) { return callback(err); } return callback(null,fileInfo); }); } }); }; /* Cleanup old files on disk, by comparing the options values: adaptorInfo from $tw.syncer.tiddlerInfo bootInfo from $tw.boot.files */ exports.cleanupTiddlerFiles = function(options,callback) { var adaptorInfo = options.adaptorInfo || {}, bootInfo = options.bootInfo || {}, title = options.title || "undefined"; if(adaptorInfo.filepath && bootInfo.filepath && adaptorInfo.filepath !== bootInfo.filepath) { $tw.utils.deleteTiddlerFile(adaptorInfo,function(err) { if(err) { if ((err.code == "EPERM" || err.code == "EACCES") && err.syscall == "unlink") { // Error deleting the previous file on disk, should fail gracefully $tw.syncer.displayError("Server desynchronized. Error cleaning up previous file for tiddler: \""+title+"\"",err); return callback(null,bootInfo); } else { return callback(err); } } return callback(null,bootInfo); }); } else { return callback(null,bootInfo); } }; })();