Added support for a serverside tiddler file store

Preparatory to implementing saving changes to the server

js/FileStore.js

@@ -0,0 +1,82 @@
+/*\
+title: js/FileStore.js
+
+\*/
+(function(){
+
+/*jslint node: true */
+"use strict";
+
+var retrieveFile = require("./FileRetriever.js").retrieveFile,
+	fs = require("fs"),
+	path = require("path"),
+	url = require("url"),
+	util = require("util"),
+	async = require("async");
+
+function FileStore(dirpath,store,callback) {
+	this.dirpath = dirpath;
+	this.store = store;
+	this.callback = callback;
+	var self = this;
+	// Set up a queue for loading tiddler files
+	this.loadQueue = async.queue(function(task,callback) {
+		retrieveFile(task.filepath,self.dirpath,function(err,data) {
+			if(err) {
+				callback(err);
+			} else {
+				// Use the filepath as the default title and src for the tiddler
+				var fields = {
+					title: data.path,
+					src: data.path
+				};
+				var tiddlers = self.store.deserializeTiddlers(data.extname,data.text,fields);
+				// Check for the .meta file
+				if(data.extname !== ".json" && tiddlers.length === 1) {
+					var metafile = task.filepath + ".meta";
+					retrieveFile(metafile,self.dirpath,function(err,data) {
+						if(err && err.code !== "ENOENT" && err.code !== "404") {
+							callback(err);
+						} else {
+							var fields = tiddlers[0];
+							if(!err) {
+								var text = data.text.split("\n\n")[0];
+								if(text) {
+									fields = self.store.deserializeTiddlers("application/x-tiddler",text,fields)[0];
+								}
+							}
+							self.store.addTiddler(fields);
+							callback(null);
+						}
+					});
+				} else {
+					self.store.addTiddlers(tiddlers);
+					callback(null);
+				}
+			}
+		});
+	},10);
+	// Call the callback when all the files are loaded
+	this.loadQueue.drain = function() {
+		callback(null);
+	};
+	// Query the folder content to get all the tiddlers
+	fs.readdir(this.dirpath,function(err,files) {
+		if(err) {
+			callback(err);
+		} else {
+			for(var t=0; t<files.length; t++) {
+				var f = files[t];
+				if(f !== ".." && f !== "." && f.indexOf(".meta") !== f.length-5) {
+					self.loadQueue.push({
+						filepath: path.resolve(self.dirpath,f)
+					});
+				}
+			}
+		}
+	});
+}
+
+exports.FileStore = FileStore;
+
+})();

js/Recipe.js

@@ -164,7 +164,7 @@ Recipe.prototype.chooseTiddlers = function(recipe) {
 			this.chooseTiddlers(recipeLine);
 		} else {
 			// Choose the store and marker array to be used for this marker
-			var store = recipeLine.marker === "shadow" ? this.store.shadows : this.store,
+			var store = recipeLine.marker === "tiddler" ? this.store : this.store.shadows,
 				markerArray = this.markers[recipeLine.marker];
 			// Create the marker array if necessary
 			if(markerArray === undefined) {
@@ -322,12 +322,19 @@ Recipe.prototype.cook = function() {
 
 // Output all the tiddlers in the recipe with a particular marker
 Recipe.prototype.outputTiddlersForMarker = function(out,marker) {
-	var tiddlers = this.markers[marker],
+	var tiddlers = [],
 		outputType = Recipe.tiddlerOutputMapper[marker] || "raw",
 		outputter = Recipe.tiddlerOutputter[outputType];
-	if(!tiddlers) {
+	if(this.markers[marker]) {
-		tiddlers = [];
+		tiddlers = this.markers[marker];
 	}
+	if(marker === "tiddler") {
+		this.store.forEachTiddler(function(title,tiddler) {
+			if(tiddlers.indexOf(title) === -1) {
+				tiddlers.push(title);
+			}
+		});
+	} 
 	if(outputter) {
 		if((out.length > 1) && (Recipe.compatibilityCheats[marker] === "suppressLeadingNewline")) {
 			var lastLine = out[out.length-1];

tiddlywiki.js

@@ -9,6 +9,7 @@ TiddlyWiki command line interface
 
 var App = require("./js/App.js").App,
 	WikiStore = require("./js/WikiStore.js").WikiStore,
+	FileStore = require("./js/FileStore.js").FileStore,
 	Tiddler = require("./js/Tiddler.js").Tiddler,
 	Recipe = require("./js/Recipe.js").Recipe,
 	tiddlerInput = require("./js/TiddlerInput.js"),
@@ -47,6 +48,7 @@ var parseOptions = function(args,defaultSwitch) {
 
 var switches = parseOptions(Array.prototype.slice.call(process.argv,2),"dummy"),
 	recipe = null,
+	fileStore = null,
 	lastRecipeFilepath = null,
 	currSwitch = 0;
 
@@ -106,6 +108,14 @@ var commandLineSwitches = {
 			});
 		}
 	},
+	store: {
+		args: {min: 1, max: 1},
+		handler: function(args,callback) {
+			fileStore = new FileStore(args[0],app.store,function() {
+				callback(null);
+			});
+		}
+	},
 	savewiki: {
 		args: {min: 1, max: 1},
 		handler: function(args,callback) {

tiddlywiki5/store/CommandLineInterface.tid

@@ -3,13 +3,25 @@ modifier: JeremyRuston
 
 TiddlyWiki5 can be used on the command line to perform an extensive set of operations based on RecipeFiles, TiddlerFiles and TiddlyWikiFiles.
 
-Usage:
+The command line interface for TiddlyWiki5 has been designed to support two distinct usages:
+* Cooking (or building) old 2.6.x versions of classic TiddlyWiki from the constituent JavaScript components
+* Cooking and serving TiddlyWiki5 itself
+
+!!Usage
 `
 node tiddlywiki.js <options>
 `
-The command line options are processed sequentially from left to right. Processing pauses during long operations, like loading a [[recipe file|RecipeFiles]] and all the subrecipes and [[tiddlers|TiddlerFiles]] that it references, and then resumes with the next command line option in sequence. The following options are available:
+The command line options are processed sequentially from left to right. Processing pauses during long operations, like loading a [[recipe file|RecipeFiles]] and all the subrecipes and [[tiddlers|TiddlerFiles]] that it references, and then resumes with the next command line option in sequence.
+
+The state that is carried between options is as follows:
+* The set of tiddlers that are currently loaded into memory
+* The recipe file last used to load tiddlers (recipe files are used primarily to manage shadow tiddlers)
+* The TiddlerFileStore used to store the main content tiddlers. This is independent of the current recipe
+
+The following options are available:
 |`--recipe <filepath>` |Loads a specfied `.recipe` file |
 |`--load <filepath>` |Load additional tiddlers from 2.x.x TiddlyWiki files (`.html`), `.tiddler`, `.tid`, `.json` or other files |
+|`--store <dirpath>` |Load a specified TiddlerFileStore |
 |`--savewiki <dirpath>` |Saves all the loaded tiddlers as a single file TiddlyWiki called `index.html` and an RSS feed called `index.xml` in a new directory of the specified name |
 |`--savetiddler <title> <filename> [<type>]` |Save an individual tiddler as a specified MIME type, defaults to `text/html` |
 |`--savetiddlers <outdir>` |Saves all the loaded tiddlers as `.tid` files in the specified directory |
@@ -34,6 +46,6 @@ This example ginsus a TiddlyWiki into its constituent tiddlers:
 node tiddlywiki.js --load mywiki.html --savetiddlers tmp/tiddlers
 `
 !! Notes
-`--servewiki` and `--servertiddlers` are for different purposes and should not be used together. The former is for TiddlyWiki core developers who want to be able to edit the TiddlyWiki source files in a text editor and view the results in the browser by clicking refresh; it is slow because it reloads all the TiddlyWiki JavaScript files each time the page is loaded. The latter is for experimenting with the new wikification engine.
+`--servewiki` and `--servetiddlers` are for different purposes and should not be used together. The former is for TiddlyWiki core developers who want to be able to edit the TiddlyWiki source files in a text editor and view the results in the browser by clicking refresh; it is slow because it reloads all the TiddlyWiki JavaScript files each time the page is loaded. The latter is for experimenting with the new wikification engine.
 
-`--wikitest` looks for `*.tid` files in the specified folder. It then wikifies the tiddlers to both "text/plain" and "text/html" format and checks the results against the content of the `*.html` and `*.txt` files in the same directory.
+`--wikitest` looks for `*.tid` files in the specified folder. It then wikifies the tiddlers to both "text/plain" and "text/html" format and checks the results against the content of the `*.html` and `*.txt` files in the same directory.

tiddlywiki5/store/Testing.tid

@@ -2,7 +2,7 @@ title: Testing
 
 !Test Scripts
 
-Three test scripts are provided, each as a Mac OS X `*.sh` bash script and a Windows `*.bat` batch file.
+Three test scripts are provided, each as a Mac OS X `*.sh` bash script and a Windows `*.bat` batch file. In each case they should be run with the current directory set to the directory in which they reside.
 
 * `test.sh`/`test.bat` cooks the main tiddlywiki.com recipe for TiddlyWiki 2.6.5 and compares it with the results of the old build process (ie, running cook.rb and then opening the file in a browser and performing a 'save changes' operation). It also runs a series of wikifications tests that work off the data in `test/wikitests/`.
 * `tw5.sh`/`tw5.bat` builds TiddlyWiki5 as a static HTML file

tiddlywiki5/tiddlywiki5.recipe

@@ -2,10 +2,6 @@ template: tiddlywiki5.template.html
 copyright: ../copyright.txt
 style: shadows/css/*.css
 
-tiddler: tiddlers/*.tid
-tiddler: tiddlers/*.jpg
-tiddler: tiddlers/*.png
-tiddler: tiddlers/*.svg
 shadow: shadows/*.tid
 shadow: shadows/templates/*.tid
 #tiddler: http://wikitext.tiddlyspace.com/fractalveg.jpg

tw5.bat

@@ -1,3 +1,3 @@
 mkdir tmp\tw5
 
-node tiddlywiki.js --recipe tiddlywiki5\tiddlywiki5.recipe --savewiki tmp\tw5 --savetiddler ReadMe readme.md
+node tiddlywiki.js --recipe tiddlywiki5\tiddlywiki5.recipe --store tiddlywiki5\store --savewiki tmp\tw5 --savetiddler ReadMe readme.md

tw5.sh

@@ -7,7 +7,7 @@ mkdir -p tmp
 mkdir -p tmp/tw5
 
 # cook TiddlyWiki5
-node tiddlywiki.js --recipe $PWD/tiddlywiki5/tiddlywiki5.recipe --savewiki tmp/tw5 --savetiddler ReadMe readme.md || exit 1
+node tiddlywiki.js --recipe $PWD/tiddlywiki5/tiddlywiki5.recipe --store tiddlywiki5/store --savewiki tmp/tw5 --savetiddler ReadMe readme.md || exit 1
 
 # cook a static version too
 #mkdir -p tmp/tw5/static

tw5s.bat

@@ -1 +1 @@
-node tiddlywiki.js --recipe tiddlywiki5\tiddlywiki5.recipe --servewiki 8080
+node tiddlywiki.js --recipe tiddlywiki5\tiddlywiki5.recipe --store tiddlywiki5\store --servewiki 8080

tw5s.sh

@@ -3,4 +3,4 @@
 # serve TiddlyWiki5 over HTTP
 
 # cook TiddlyWiki5
-node tiddlywiki.js --recipe $PWD/tiddlywiki5/tiddlywiki5.recipe --servewiki 8080 || exit 1
+node tiddlywiki.js --recipe $PWD/tiddlywiki5/tiddlywiki5.recipe --store tiddlywiki5/store --servewiki 8080 || exit 1