Refactored tiddler serialization and deserialization

Introduced TiddlerConverters, a sort of factory for them

js/Recipe.js

@@ -51,8 +51,6 @@ At this point tiddlers are placed in the store so that they can be referenced by
 
 var Tiddler = require("./Tiddler.js").Tiddler,
 	WikiTextRenderer = require("./WikiTextRenderer").WikiTextRenderer,
-	tiddlerInput = require("./TiddlerInput.js"),
-	tiddlerOutput = require("./TiddlerOutput.js"),
 	utils = require("./Utils.js"),
 	retrieveFile = require("./FileRetriever.js").retrieveFile,
 	fs = require("fs"),
@@ -60,11 +58,11 @@ var Tiddler = require("./Tiddler.js").Tiddler,
 	util = require("util"),
 	async = require("async");
 
-// Create a new Recipe object from the specified recipe file, storing the tiddlers in a specified TiddlyWiki store. Invoke
+var Recipe = function(options,callback) {
-// the callback function when all of the referenced tiddlers and recipes have been loaded successfully
-var Recipe = function(store,filepath,callback) {
 	var me = this;
-	this.store = store; // Save a reference to the store
+	this.filepath = options.filepath;
+	this.store = options.store;
+	this.tiddlerConverters = options.tiddlerConverters;
 	this.callback = callback;
 	this.recipe = [];
 	this.markers = {};
@@ -94,9 +92,9 @@ var Recipe = function(store,filepath,callback) {
 	this.tiddlerQueue.drain = function() {
 		me.chooseTiddlers(me.recipe);
 		me.sortTiddlersForMarker("tiddler");
-		me.callback();
+		me.callback(null);
 	};
-	this.recipeQueue.push({filepath: filepath,
+	this.recipeQueue.push({filepath: this.filepath,
 							contextPath: process.cwd(),
 							recipe: this.recipe});
 };
@@ -176,7 +174,7 @@ Recipe.prototype.readTiddlerFile = function(filepath,contextPath,callback) {
 		var fields = {
 			title: data.path
 		};
-		var tiddlers = tiddlerInput.parseTiddlerFile(data.text,data.extname,fields);
+		var tiddlers = me.tiddlerConverters.deserialize(data.extname,data.text,fields);
 		// Check for the .meta file
 		if(data.extname !== ".json" && tiddlers.length === 1) {
 			var metafile = filepath + ".meta";
@@ -186,7 +184,10 @@ Recipe.prototype.readTiddlerFile = function(filepath,contextPath,callback) {
 				} else {
 					var fields = tiddlers[0];
 					if(!err) {
-						fields = tiddlerInput.parseMetaDataBlock(data.text,fields);
+						var text = data.text.split("\n\n")[0];
+						if(text) {
+							fields = me.tiddlerConverters.deserialize("application/x-tiddler",text,fields)[0];
+						}
 					}
 					callback(null,[fields]);
 				}
@@ -252,7 +253,7 @@ Recipe.tiddlerOutputter = {
 		// Ordinary tiddlers are output as a <DIV>
 		for(var t=0; t<tiddlers.length; t++) {
 			var tid = this.store.getTiddler(tiddlers[t]);
-			out.push(tiddlerOutput.outputTiddlerDiv(tid));
+			out.push(this.tiddlerConverters.serialize("application/x-tiddler-html-div",tid));
 		}
 	},
 	javascript: function(out,tiddlers) {
@@ -275,7 +276,7 @@ Recipe.tiddlerOutputter = {
 		for(var t=0; t<tiddlers.length; t++) {
 			var title = tiddlers[t],
 				tid = this.store.shadows.getTiddler(title);
-			out.push(tiddlerOutput.outputTiddlerDiv(tid));
+			out.push(this.tiddlerConverters.serialize("application/x-tiddler-html-div",tid));
 		}
 	},
 	title: function(out,tiddlers) {

js/TiddlerConverters.js

@@ -0,0 +1,46 @@
+/*jslint node: true */
+"use strict";
+
+var TiddlerConverters = function() {
+	this.serializers = {};
+	this.deserializers = {};
+};
+
+TiddlerConverters.prototype.registerSerializer = function(extension,mimeType,serializer) {
+	this.serializers[extension] = serializer;
+	this.serializers[mimeType] = serializer;
+};
+
+TiddlerConverters.prototype.registerDeserializer = function(extension,mimeType,deserializer) {
+	this.deserializers[extension] = deserializer;
+	this.deserializers[mimeType] = deserializer;
+};
+
+TiddlerConverters.prototype.serialize = function(type,tiddler) {
+	var serializer = this.serializers[type];
+	if(serializer) {
+		return serializer(tiddler);
+	} else {
+		return null;
+	}
+};
+
+TiddlerConverters.prototype.deserialize = function(type,text,srcFields) {
+	var fields = {},
+		deserializer = this.deserializers[type],
+		t;
+	if(srcFields) {
+		for(t in srcFields) {
+			fields[t] = srcFields[t];		
+		}
+	}
+	if(deserializer) {
+		return deserializer(text,fields);
+	} else {
+		// Return a raw tiddler for unknown types
+		fields.text = text;
+		return [fields];
+	}
+};
+
+exports.TiddlerConverters = TiddlerConverters;

js/TiddlerInput.js

@@ -11,117 +11,7 @@ var utils = require("./Utils.js"),
 var tiddlerInput = exports;
 
 /*
-Parse a tiddler given its mimetype, and merge the results into a hashmap of tiddler fields.
+Utility function to parse a block of metadata and merge the results into a hashmap of tiddler fields.
-
-A file extension can be passed as a shortcut for the mimetype, as shown in tiddlerUtils.fileExtensionMappings.
-For example ".txt" file extension is mapped to the "text/plain" mimetype.
-
-Special processing to extract embedded metadata is applied to some mimetypes.
-*/
-
-tiddlerInput.parseTiddlerFile = function(text,type,fields) {
-	// Map extensions to mimetpyes
-	var fileExtensionMapping = tiddlerInput.fileExtensionMappings[type];
-	if(fileExtensionMapping)
-		type = fileExtensionMapping;
-	// Invoke the parser for the specified mimetype
-	var parser = tiddlerInput.parseTiddlerFileByMimeType[type];
-	if(!parser)
-		parser = tiddlerInput.parseTiddlerFileByMimeType["text/plain"];
-	return parser(text,fields);
-};
-
-tiddlerInput.fileExtensionMappings = {
-	".txt": "text/plain",
-	".html": "text/html",
-	".tiddler": "application/x-tiddler-html-div",
-	".tid": "application/x-tiddler",
-	".js": "application/javascript",
-	".json": "application/json",
-	".tiddlywiki": "application/x-tiddlywiki"
-};
-
-tiddlerInput.parseTiddlerFileByMimeType = {
-	"text/plain": function(text,fields) {
-		fields.text = text;
-		return [fields];
-	},
-	"text/html": function(text,fields) {
-		fields.text = text;
-		return [fields];
-	},
-	"application/x-tiddler-html-div": function(text,fields) {
-		return [tiddlerInput.parseTiddlerDiv(text,fields)];
-	},
-	"application/x-tiddler": function(text,fields) {
-		var split = text.indexOf("\n\n");
-		if(split === -1) {
-			split = text.length;
-		}
-		fields = tiddlerInput.parseMetaDataBlock(text.substr(0,split),fields);
-		fields.text = text.substr(split + 2);
-		return [fields];
-	},
-	"application/javascript": function(text,fields) {
-		fields.text = text;
-		return [fields];
-	},
-	"application/json": function(text,fields) {
-		var tiddlers = JSON.parse(text),
-			result = [],
-			getKnownFields = function(tid) {
-				var fields = {};
-				"title text created creator modified modifier type tags".split(" ").forEach(function(value) {
-					fields[value] = tid[value];
-				});
-				return fields;
-			};
-		for(var t=0; t<tiddlers.length; t++) {
-			result.push(getKnownFields(tiddlers[t]));
-		}
-		return result;
-	},
-	"application/x-tiddlywiki": function(text,fields) {
-		var results = [],
-			storeAreaPos = tiddlerInput.locateStoreArea(text);
-		if(storeAreaPos) {
-			var endOfDivRegExp = /(<\/div>\s*)/gi,
-				startPos = storeAreaPos[0];
-			endOfDivRegExp.lastIndex = startPos;
-			var match = endOfDivRegExp.exec(text);
-			while(match && startPos < storeAreaPos[1]) {
-				var endPos = endOfDivRegExp.lastIndex,
-					tiddlerFields = tiddlerInput.parseTiddlerDiv(text.substring(startPos,endPos),fields);
-				tiddlerFields.text = utils.htmlDecode(tiddlerFields.text);
-				results.push(tiddlerFields);
-				startPos = endPos;
-				match = endOfDivRegExp.exec(text);
-			}
-		}
-		return results;
-	}
-};
-
-tiddlerInput.locateStoreArea = function(tiddlywikidoc) {
-	var startSaveArea = '<div id="' + 'storeArea">',
-		startSaveAreaRegExp = /<div id=["']?storeArea['"]?>/gi,
-		endSaveArea = '</d' + 'iv>',
-		endSaveAreaCaps = '</D' + 'IV>',
-		posOpeningDiv = tiddlywikidoc.search(startSaveAreaRegExp),
-		limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-STOREAREA--"+">");
-	if(limitClosingDiv == -1) {
-		limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-BODY-START--"+">");
-	}
-	var start = limitClosingDiv == -1 ? tiddlywikidoc.length : limitClosingDiv,
-		posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveArea,start);
-	if(posClosingDiv == -1) {
-		posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveAreaCaps,start);
-	}
-	return (posOpeningDiv != -1 && posClosingDiv != -1) ? [posOpeningDiv + startSaveArea.length,posClosingDiv] : null;
-};
-
-/*
-Parse a block of metadata and merge the results into a hashmap of tiddler fields.
 
 The block consists of newline delimited lines consisting of the field name, a colon, and then the value. For example:
 
@@ -132,7 +22,7 @@ modified: 20110211131020
 tags: browsers issues
 creator: psd
 */
-tiddlerInput.parseMetaDataBlock = function(metaData,fields) {
+var parseMetaDataBlock = function(metaData,fields) {
 	var result = {};
 	if(fields) {
 		for(var t in fields) {
@@ -151,7 +41,7 @@ tiddlerInput.parseMetaDataBlock = function(metaData,fields) {
 };
 
 /*
-Parse an old-style tiddler DIV. It looks like this:
+Utility function to parse an old-style tiddler DIV. It looks like this:
 
 <div title="Title" creator="JoeBloggs" modifier="JoeBloggs" created="201102111106" modified="201102111310" tags="myTag [[my long tag]]">
 <pre>The text of the tiddler (without the expected HTML encoding).
@@ -160,7 +50,7 @@ Parse an old-style tiddler DIV. It looks like this:
 
 Note that the field attributes are HTML encoded, but that the body of the <PRE> tag is not.
 */
-tiddlerInput.parseTiddlerDiv = function(text,fields) {
+var parseTiddlerDiv = function(text,fields) {
 	var result = {};
 	if(fields) {
 		for(var t in fields) {
@@ -190,3 +80,85 @@ tiddlerInput.parseTiddlerDiv = function(text,fields) {
 	}
 	return result;	
 };
+
+var inputTiddlerPlain = function(text,fields) {
+	fields.text = text;
+	return [fields];
+};
+
+var inputTiddlerDiv = function(text,fields) {
+	return [parseTiddlerDiv(text,fields)];
+};
+
+var inputTiddler = function(text,fields) {
+	var split = text.indexOf("\n\n");
+	if(split !== -1) {
+		fields.text = text.substr(split + 2);
+	}
+	if(split === -1) {
+		split = text.length;
+	}
+	fields = parseMetaDataBlock(text.substr(0,split),fields);
+	return [fields];
+};
+
+var inputTiddlerJSON = function(text,fields) {
+	var tiddlers = JSON.parse(text),
+		result = [],
+		getKnownFields = function(tid) {
+			var fields = {};
+			"title text created creator modified modifier type tags".split(" ").forEach(function(value) {
+				fields[value] = tid[value];
+			});
+			return fields;
+		};
+	for(var t=0; t<tiddlers.length; t++) {
+		result.push(getKnownFields(tiddlers[t]));
+	}
+	return result;
+};
+
+var inputTiddlyWiki = function(text,fields) {
+	var locateStoreArea = function(tiddlywikidoc) {
+			var startSaveArea = '<div id="' + 'storeArea">',
+				startSaveAreaRegExp = /<div id=["']?storeArea['"]?>/gi,
+				endSaveArea = '</d' + 'iv>',
+				endSaveAreaCaps = '</D' + 'IV>',
+				posOpeningDiv = tiddlywikidoc.search(startSaveAreaRegExp),
+				limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-STOREAREA--"+">");
+			if(limitClosingDiv == -1) {
+				limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-BODY-START--"+">");
+			}
+			var start = limitClosingDiv == -1 ? tiddlywikidoc.length : limitClosingDiv,
+				posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveArea,start);
+			if(posClosingDiv == -1) {
+				posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveAreaCaps,start);
+			}
+			return (posOpeningDiv != -1 && posClosingDiv != -1) ? [posOpeningDiv + startSaveArea.length,posClosingDiv] : null;
+		},
+		results = [],
+		storeAreaPos = locateStoreArea(text);
+	if(storeAreaPos) {
+		var endOfDivRegExp = /(<\/div>\s*)/gi,
+			startPos = storeAreaPos[0];
+		endOfDivRegExp.lastIndex = startPos;
+		var match = endOfDivRegExp.exec(text);
+		while(match && startPos < storeAreaPos[1]) {
+			var endPos = endOfDivRegExp.lastIndex,
+				tiddlerFields = parseTiddlerDiv(text.substring(startPos,endPos),fields);
+			tiddlerFields.text = utils.htmlDecode(tiddlerFields.text);
+			results.push(tiddlerFields);
+			startPos = endPos;
+			match = endOfDivRegExp.exec(text);
+		}
+	}
+	return results;
+};
+
+tiddlerInput.register = function(tiddlerConverters) {
+	tiddlerConverters.registerDeserializer(".txt","text/plain",inputTiddlerPlain);
+	tiddlerConverters.registerDeserializer(".tiddler","application/x-tiddler-html-div",inputTiddlerDiv);
+	tiddlerConverters.registerDeserializer(".tid","application/x-tiddler",inputTiddler);
+	tiddlerConverters.registerDeserializer(".json","application/json",inputTiddlerJSON);
+	tiddlerConverters.registerDeserializer(".tiddlywiki","application/x-tiddlywiki",inputTiddlyWiki);
+};

js/TiddlerOutput.js

@@ -5,14 +5,28 @@ Functions concerned with parsing representations of tiddlers
 /*jslint node: true */
 "use strict";
 
-var utils = require("./Utils.js");
+var utils = require("./Utils.js"),
+	util = require("util");
 
 var tiddlerOutput = exports;
 
+// Utility function to convert a tags string array into a TiddlyWiki-style quoted tags string
+var stringifyTags = function(tags) {
+	var results = [];
+	for(var t=0; t<tags.length; t++) {
+		if(tags[t].indexOf(" ") !== -1) {
+			results.push("[[" + tags[t] + "]]");
+		} else {
+			results.push(tags[t]);
+		}
+	}
+	return results.join(" ");
+};
+
 /*
 Output a tiddler as a .tid file
 */
-tiddlerOutput.outputTiddler = function(tid) {
+var outputTiddler = function(tid) {
 	var result = [],
 		outputAttribute = function(name,value) {
 		result.push(name + ": " + value + "\n");
@@ -48,7 +62,7 @@ out - array to push the output strings
 tid - the tiddler to be output
 The fields are in the order title, creator, modifier, created, modified, tags, followed by any others
 */
-tiddlerOutput.outputTiddlerDiv = function(tid) {
+var outputTiddlerDiv = function(tid) {
 	var result = [],
 		attributes = {},
 		outputAttribute = function(name,transform,dontDelete) {
@@ -75,7 +89,7 @@ tiddlerOutput.outputTiddlerDiv = function(tid) {
 	outputAttribute("modifier");
 	outputAttribute("created", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
 	outputAttribute("modified", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
-	outputAttribute("tags", function(v) {return tiddlerOutput.stringifyTags(v);});
+	outputAttribute("tags", function(v) {return stringifyTags(v);});
 	// Output any other attributes
 	for(t in attributes) {
 		outputAttribute(t,null,true);
@@ -86,16 +100,7 @@ tiddlerOutput.outputTiddlerDiv = function(tid) {
 	return result.join("");
 };
 
-tiddlerOutput.stringifyTags = function(tags) {
+tiddlerOutput.register = function(tiddlerConverters) {
-	var results = [];
+	tiddlerConverters.registerSerializer(".tid","application/x-tiddler",outputTiddler);
-	for(var t=0; t<tags.length; t++) {
+	tiddlerConverters.registerSerializer(".tiddler","application/x-tiddler-html-div",outputTiddlerDiv);
-		if(tags[t].indexOf(" ") !== -1) {
-			results.push("[[" + tags[t] + "]]");
-		} else {
-			results.push(tags[t]);
-		}
-	}
-	return results.join(" ");
 };
-
-

tiddlywiki.js

@@ -9,8 +9,9 @@ var WikiStore = require("./js/WikiStore.js").WikiStore,
 	Tiddler = require("./js/Tiddler.js").Tiddler,
 	Recipe = require("./js/Recipe.js").Recipe,
 	Tiddler = require("./js/Tiddler.js").Tiddler,
-	tiddlerOutput = require("./js/TiddlerOutput.js"),
 	tiddlerInput = require("./js/TiddlerInput.js"),
+	tiddlerOutput = require("./js/TiddlerOutput.js"),
+	TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters,
 	util = require("util"),
 	fs = require("fs"),
 	url = require("url"),
@@ -41,12 +42,17 @@ var parseOptions = function(args,defaultSwitch) {
 	return result;
 };
 
-var switches = parseOptions(Array.prototype.slice.call(process.argv,2),"dummy"),
+var tiddlerConverters = new TiddlerConverters(),
+	switches = parseOptions(Array.prototype.slice.call(process.argv,2),"dummy"),
 	store = new WikiStore(),
 	recipe = null,
 	lastRecipeFilepath = null,
 	currSwitch = 0;
 
+// Register the standard tiddler serializers and deserializers
+tiddlerInput.register(tiddlerConverters);
+tiddlerOutput.register(tiddlerConverters);
+
 // Add the shadow tiddlers that are built into TiddlyWiki
 var shadowShadowStore = new WikiStore(null),
 	shadowShadows = [
@@ -114,7 +120,11 @@ var commandLineSwitches = {
 				callback("--recipe: Cannot process more than one recipe file");
 			} else {
 				lastRecipeFilepath = args[0];
-				recipe = new Recipe(store,args[0],function() {
+				recipe = new Recipe({
+					filepath: args[0],
+					store: store,
+					tiddlerConverters: tiddlerConverters
+				},function() {
 					callback(null);
 				});
 			}

wikitest.js

@@ -15,22 +15,26 @@ verifying that the output matches `<tiddlername>.html` and `<tiddlername>.txt`.
 var Tiddler = require("./js/Tiddler.js").Tiddler,
 	WikiStore = require("./js/WikiStore.js").WikiStore,
 	WikiTextRenderer = require("./js/WikiTextRenderer.js").WikiTextRenderer,
-	tiddlerInput = require("./js/TiddlerInput"),
+	TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters,
+	tiddlerInput = require("./js/TiddlerInput.js"),
 	utils = require("./js/Utils.js"),
 	util = require("util"),
 	fs = require("fs"),
 	path = require("path");
 
 var testdirectory = process.argv[2],
+	tiddlerConverters = new TiddlerConverters(),
 	store = new WikiStore(),
 	files = fs.readdirSync(testdirectory),
 	titles = [],
 	f,t,extname,basename;
 
+tiddlerInput.register(tiddlerConverters);
+
 for(f=0; f<files.length; f++) {
 	extname = path.extname(files[f]);
 	if(extname === ".tid") {
-		var tiddlers = tiddlerInput.parseTiddlerFile(fs.readFileSync(path.resolve(testdirectory,files[f]),"utf8"),extname);
+		var tiddlers = tiddlerConverters.deserialize(extname,fs.readFileSync(path.resolve(testdirectory,files[f]),"utf8"));
 		if(tiddlers.length > 1) {
 			throw "Cannot use .JSON files";
 		}