Major refactoring of how wiki text parsing and rendering is packaged

js/Recipe.js

@@ -50,7 +50,6 @@ At this point tiddlers are placed in the store so that they can be referenced by
 "use strict";
 
 var Tiddler = require("./Tiddler.js").Tiddler,
-	WikiTextRenderer = require("./WikiTextRenderer").WikiTextRenderer,
 	utils = require("./Utils.js"),
 	retrieveFile = require("./FileRetriever.js").retrieveFile,
 	fs = require("fs"),
@@ -63,6 +62,7 @@ var Recipe = function(options,callback) {
 	this.filepath = options.filepath;
 	this.store = options.store;
 	this.tiddlerConverters = options.tiddlerConverters;
+	this.textProcessors = options.textProcessors;
 	this.callback = callback;
 	this.recipe = [];
 	this.markers = {};
@@ -280,7 +280,7 @@ Recipe.tiddlerOutputter = {
 		}
 	},
 	title: function(out,tiddlers) {
-		out.push(this.renderTiddler("WindowTitle","text/plain"));
+		out.push(this.store.renderTiddler("text/plain","WindowTitle"));
 	}
 };
 
@@ -291,13 +291,13 @@ Recipe.prototype.cookRss = function()
 		numRssItems = 20,
 		s = [],
 		d = new Date(),
-		u = this.renderTiddler("SiteUrl","text/plain"),
+		u = this.store.renderTiddler("text/plain","SiteUrl"),
 		encodeTiddlyLink = function(title) {
 			return title.indexOf(" ") == -1 ? title : "[[" + title + "]]";
 		},
 		tiddlerToRssItem = function(tiddler,uri) {
 			var s = "<title" + ">" + utils.htmlEncode(tiddler.fields.title) + "</title" + ">\n";
-			s += "<description>" + utils.htmlEncode(me.renderTiddler(tiddler.fields.title,"text/plain")) + "</description>\n";
+			s += "<description>" + utils.htmlEncode(me.store.renderTiddler("text/plain",tiddler.fields.title)) + "</description>\n";
 			var i;
 			if(tiddler.fields.tags) {
 				for(i=0; i<tiddler.fields.tags.length; i++) {
@@ -336,10 +336,10 @@ Recipe.prototype.cookRss = function()
 	s.push("<" + "?xml version=\"1.0\"?" + ">");
 	s.push("<rss version=\"2.0\">");
 	s.push("<channel>");
-	s.push("<title" + ">" + utils.htmlEncode(this.renderTiddler("SiteTitle","text/plain")) + "</title" + ">");
+	s.push("<title" + ">" + utils.htmlEncode(this.store.renderTiddler("text/plain","SiteTitle")) + "</title" + ">");
 	if(u)
 		s.push("<link>" + utils.htmlEncode(u) + "</link>");
-	s.push("<description>" + utils.htmlEncode(this.renderTiddler("SiteSubtitle","text/plain")) + "</description>");
+	s.push("<description>" + utils.htmlEncode(this.store.renderTiddler("text/plain","SiteSubtitle")) + "</description>");
 	//s.push("<language>" + config.locale + "</language>");
 	s.push("<pubDate>" + d.toUTCString() + "</pubDate>");
 	s.push("<lastBuildDate>" + d.toUTCString() + "</lastBuildDate>");
@@ -358,10 +358,5 @@ Recipe.prototype.cookRss = function()
 	return s.join("\n");
 };
 
-Recipe.prototype.renderTiddler = function(title,type) {
-	var r = new WikiTextRenderer(this.store.getTiddler(title).getParseTree(),this.store,title);
-	return r.render(type);
-};
-
 exports.Recipe = Recipe;
 

js/TextProcessors.js

@@ -0,0 +1,26 @@
+/*jslint node: true */
+"use strict";
+
+var util = require("util");
+
+var TextProcessors = function() {
+	this.processors = {};
+};
+
+TextProcessors.prototype.registerTextProcessor = function(type,processor) {
+	this.processors[type] = processor;
+};
+
+TextProcessors.prototype.parse = function(type,text) {
+	var processor = this.processors[type];
+	if(!processor) {
+		processor = this.processors["text/x-tiddlywiki"];
+	}
+	if(processor) {
+		return processor.parse(text);
+	} else {
+		return null;
+	}
+};
+
+exports.TextProcessors = TextProcessors;

js/Tiddler.js

@@ -116,27 +116,4 @@ Tiddler.specialTiddlerFieldParsers = {
 	}
 };
 
-Tiddler.prototype.getParseTree = function() {
-	if(!this.parseTree) {
-		var type = this.fields.type || "application/x-tiddlywikitext",
-		parser = Tiddler.tiddlerTextParsers[type];
-		if(parser) {
-			this.parseTree = Tiddler.tiddlerTextParsers[type].call(this);
-		}
-	}
-	return this.parseTree;
-};
-
-Tiddler.tiddlerTextParsers = {
-	"application/x-tiddlywikitext": function() {
-		return new WikiTextParser(this.fields.text);
-	},
-	"application/javascript": function() {
-		// Would be useful to parse so that we can do syntax highlighting and debugging
-	},
-	"application/json": function() {
-		return JSON.parse(this.fields.text);
-	}
-};
-
 exports.Tiddler = Tiddler;

js/WikiStore.js

@@ -4,9 +4,13 @@
 var Tiddler = require("./Tiddler.js").Tiddler,
 	util = require("util");
 
-var WikiStore = function WikiStore(shadowStore) {
+var WikiStore = function WikiStore(options) {
 	this.tiddlers = {};
-	this.shadows = shadowStore === undefined ? new WikiStore(null) : shadowStore;
+	this.shadows = options.shadowStore !== undefined ? options.shadowStore : new WikiStore({
+		shadowStore: null,
+		textProcessors: options.textProcessors
+	});
+	this.textProcessors = options.textProcessors;
 };
 
 WikiStore.prototype.clear = function() {
@@ -50,4 +54,22 @@ WikiStore.prototype.forEachTiddler = function(callback) {
 	}
 };
 
+WikiStore.prototype.parseTiddler = function(title) {
+	var tiddler = this.getTiddler(title);
+	if(tiddler) {
+		return this.textProcessors.parse(tiddler.fields.type,tiddler.fields.text);
+	} else {
+		return null;
+	}
+}
+
+WikiStore.prototype.renderTiddler = function(type,title) {
+	var parser = this.parseTiddler(title);
+	if(parser) {
+		return parser.render(type,parser.children,this,title);
+	} else {
+		return null;
+	}
+}
+
 exports.WikiStore = WikiStore;

js/WikiTextParser.js

@@ -26,17 +26,18 @@ Text nodes are:
 /*jslint node: true */
 "use strict";
 
-var wikiTextRules = require("./WikiTextRules.js").wikiTextRules,
+var WikiTextRenderer = require("./WikiTextRenderer.js").WikiTextRenderer,
 	utils = require("./Utils.js"),
 	util = require("util");
 
-var WikiTextParser = function(text) {
+var WikiTextParser = function(text,processor) {
+	this.processor = processor;
 	this.autoLinkWikiWords = true;
 	this.source = text;
 	this.nextMatch = 0;
-	this.tree = [];
+	this.children = [];
 	this.output = null;
-	this.subWikify(this.tree);
+	this.subWikify(this.children);
 };
 
 WikiTextParser.prototype.outputText = function(place,startPos,endPos) {
@@ -58,8 +59,8 @@ WikiTextParser.prototype.subWikifyUnterm = function(output) {
 	var oldOutput = this.output;
 	this.output = output;
 	// Get the first match
-	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	this.processor.rulesRegExp.lastIndex = this.nextMatch;
-	var ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+	var ruleMatch = this.processor.rulesRegExp.exec(this.source);
 	while(ruleMatch) {
 		// Output any text before the match
 		if(ruleMatch.index > this.nextMatch)
@@ -68,18 +69,18 @@ WikiTextParser.prototype.subWikifyUnterm = function(output) {
 		this.matchStart = ruleMatch.index;
 		this.matchLength = ruleMatch[0].length;
 		this.matchText = ruleMatch[0];
-		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		this.nextMatch = this.processor.rulesRegExp.lastIndex;
 		// Figure out which rule matched and call its handler
 		var t;
 		for(t=1; t<ruleMatch.length; t++) {
 			if(ruleMatch[t]) {
-				wikiTextRules.rules[t-1].handler(this);
+				this.processor.rules[t-1].handler(this);
-				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				this.processor.rulesRegExp.lastIndex = this.nextMatch;
 				break;
 			}
 		}
 		// Get the next match
-		ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+		ruleMatch = this.processor.rulesRegExp.exec(this.source);
 	}
 	// Output any text after the last match
 	if(this.nextMatch < this.source.length) {
@@ -97,8 +98,8 @@ WikiTextParser.prototype.subWikifyTerm = function(output,terminatorRegExp) {
 	// Get the first matches for the rule and terminator RegExps
 	terminatorRegExp.lastIndex = this.nextMatch;
 	var terminatorMatch = terminatorRegExp.exec(this.source);
-	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	this.processor.rulesRegExp.lastIndex = this.nextMatch;
-	var ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+	var ruleMatch = this.processor.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
 	while(terminatorMatch || ruleMatch) {
 		// Check for a terminator match before the next rule match
 		if(terminatorMatch && (!ruleMatch || terminatorMatch.index <= ruleMatch.index)) {
@@ -121,20 +122,20 @@ WikiTextParser.prototype.subWikifyTerm = function(output,terminatorRegExp) {
 		this.matchStart = ruleMatch.index;
 		this.matchLength = ruleMatch[0].length;
 		this.matchText = ruleMatch[0];
-		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		this.nextMatch = this.processor.rulesRegExp.lastIndex;
 		// Figure out which rule matched and call its handler
 		var t;
 		for(t=1; t<ruleMatch.length; t++) {
 			if(ruleMatch[t]) {
-				wikiTextRules.rules[t-1].handler(this);
+				this.processor.rules[t-1].handler(this);
-				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				this.processor.rulesRegExp.lastIndex = this.nextMatch;
 				break;
 			}
 		}
 		// Get the next match
 		terminatorRegExp.lastIndex = this.nextMatch;
 		terminatorMatch = terminatorRegExp.exec(this.source);
-		ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+		ruleMatch = this.processor.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
 	}
 	// Output any text after the last match
 	if(this.nextMatch < this.source.length) {
@@ -145,4 +146,9 @@ WikiTextParser.prototype.subWikifyTerm = function(output,terminatorRegExp) {
 	this.output = oldOutput;
 };
 
+WikiTextParser.prototype.render = function(type,treenode,store,title) {
+	var renderer = new WikiTextRenderer(store,title,this);
+	return renderer.render(type,treenode);
+};
+
 exports.WikiTextParser = WikiTextParser;

js/WikiTextProcessor.js

@@ -0,0 +1,30 @@
+/*jslint node: true */
+"use strict";
+
+var WikiTextRules = require("./WikiTextRules.js"),
+	WikiTextParser = require("./WikiTextParser.js").WikiTextParser;
+
+
+/*
+Creates a new instance of the wiki text processor with the specified options. The
+options are a hashmap of optional members as follows:
+
+	enableRules: An array of names of wiki text rules to enable. If not specified, all rules are available
+	extraRules: An array of additional rule handlers to add
+	enableMacros: An array of names of macros to enable. If not specified, all macros are available
+	extraMacros: An array of additional macro handlers to add
+*/
+var WikiTextProcessor = function(options) {
+	this.rules = WikiTextRules.rules;
+	var pattern = [];
+	for(var n=0; n<this.rules.length; n++) {
+		pattern.push("(" + this.rules[n].match + ")");
+	}
+	this.rulesRegExp = new RegExp(pattern.join("|"),"mg");
+};
+
+WikiTextProcessor.prototype.parse = function(text) {
+	return new WikiTextParser(text,this);
+}
+
+exports.WikiTextProcessor = WikiTextProcessor;

js/WikiTextRenderer.js

@@ -6,27 +6,27 @@ Wiki text macro implementation
 "use strict";
 
 var ArgParser = require("./ArgParser.js").ArgParser,
-	WikiTextParser = require("./WikiTextParser.js").WikiTextParser,
+	WikiTextParserModule = require("./WikiTextParser.js"),
 	utils = require("./Utils.js"),
 	util = require("util");
 
-var WikiTextRenderer = function(parser,store,title) {
+var WikiTextRenderer = function(store,title,parser) {
 	this.parser = parser;
 	this.store = store;
 	this.title = title;
 };
 
-WikiTextRenderer.prototype.render = function(type) {
+WikiTextRenderer.prototype.render = function(type,treenode) {
 	if(type === "text/html") {
-		return this.renderAsHtml();
+		return this.renderAsHtml(treenode);
 	} else if (type === "text/plain") {
-		return this.renderAsText();
+		return this.renderAsText(treenode);
 	} else {
 		return null;
 	}
 };
 
-WikiTextRenderer.prototype.renderAsHtml = function() {
+WikiTextRenderer.prototype.renderAsHtml = function(treenode) {
 	var output = [],
 		renderSubTree;
 	var renderElement = function(element, selfClosing) {
@@ -74,12 +74,12 @@ WikiTextRenderer.prototype.renderAsHtml = function() {
 			}
 		}
 	};
-	this.executeMacros(this.parser.tree);
+	this.executeMacros(treenode);
-	renderSubTree(this.parser.tree);
+	renderSubTree(treenode);
 	return output.join("");	
 };
 
-WikiTextRenderer.prototype.renderAsText = function() {
+WikiTextRenderer.prototype.renderAsText = function(treenode) {
 	var output = [];
 	var renderSubTree = function(tree) {
 		for(var t=0; t<tree.length; t++) {
@@ -104,8 +104,8 @@ WikiTextRenderer.prototype.renderAsText = function() {
 			}
 		}
 	};
-	this.executeMacros(this.parser.tree);
+	this.executeMacros(treenode);
-	renderSubTree(this.parser.tree);
+	renderSubTree(treenode);
 	return output.join("");	
 };
 
@@ -176,9 +176,9 @@ WikiTextRenderer.macros = {
 				var placeholderRegExp = new RegExp("\\$"+(t+1),"mg");
 				text = text.replace(placeholderRegExp,withTokens[t]);
 			}
-			var parseTree = new WikiTextParser(text);
+			var parseTree = new WikiTextParserModule.WikiTextParser(text,this.parser.processor);
-			for(t=0; t<parseTree.tree.length; t++) {
+			for(t=0; t<parseTree.children.length; t++) {
-				macroNode.output.push(parseTree.tree[t]);
+				macroNode.output.push(parseTree.children[t]);
 			}
 			// Execute any macros in the copy
 			this.executeMacros(macroNode.output);

js/WikiTextRules.js

@@ -37,16 +37,6 @@ textPrimitives.tiddlerAnyLinkRegExp = new RegExp("("+ textPrimitives.wikiLink +
 	textPrimitives.brackettedLink + ")|(?:" +
 	textPrimitives.urlPattern + ")","mg");
 
-function WikiTextRules()
-{
-	var pattern = [];
-	this.rules = WikiTextRules.rules;
-	for(var n=0; n<this.rules.length; n++) {
-		pattern.push("(" + this.rules[n].match + ")");
-	}
-	this.rulesRegExp = new RegExp(pattern.join("|"),"mg");
-}
-
 var createElementAndWikify = function(w) {
 	var e = {type: this.element, children: []};
 	w.output.push(e);
@@ -111,7 +101,7 @@ var enclosedTextHelper = function(w) {
 	}
 };
 
-WikiTextRules.rules = [
+var rules = [
 {
 	name: "table",
 	match: "^\\|(?:[^\\n]*)\\|(?:[fhck]?)$",
@@ -676,4 +666,4 @@ WikiTextRules.rules = [
 
 ];
 
-exports.wikiTextRules = new WikiTextRules();
+exports.rules = rules;

readme.md

@@ -46,6 +46,262 @@ You can use filepaths or URLs to reference recipe files and tiddlers. For exampl
 
 `test.sh` contains a simple test that cooks the main tiddlywiki.com recipe and compares it with the results of the old build process (ie, running cook.rb and then opening the file in a browser and performing a 'save changes' operation). It also invokes `wikitest.js`, a wikification test rig that works off the data in `test/wikitests/`.
 
-## Current status
+## API
 
-As of 8th December 2011, cook.js can now build a fully functional TiddlyWiki and its RSS feed from the existing recipe files. There are two or three minor whitespace issues that prevent full byte-for-byte compatibility.
+Here is a guide to the key modules making up tiddlywiki.js and their public APIs. The modules are listed in order of dependency; modules generally don't know about other modules later in the list unless specifically noted.
+
+Some non-standard MIME types are used by the code:
+
+* **text/x-tiddlywiki:** TiddlyWiki-format wiki text
+* **application/x-tiddlywiki:** A TiddlyWiki HTML file containing tiddlers
+* **application/x-tiddler:** A tiddler in TiddlyWeb-style tiddler file format
+* **application/x-tiddler-html-div:** A tiddler in TiddlyWiki `<div>` format
+
+### Tiddler.js
+
+Tiddlers are an immutable dictionary of name:value pairs called fields. Values can be a string, an array of strings, or a JavaScript date object.
+
+The only field that is required is the `title` field, but useful tiddlers also have a `text` field, and some or all of the standard fields `modified`, `modifier`, `created`, `creator`, `tags` and `type`.
+
+Hardcoded in the system is the knowledge that the 'tags' field is a string array, and that the 'modified' and 'created' fields are dates. All other fields are strings.
+
+#### var tiddler = new Tiddler([srcFields{,srcFields}])
+
+Create a Tiddler given a series of sources of fields which can either be a plain hashmap of name:value pairs or an existing tiddler to clone. Fields in later sources overwrite the same field specified in earlier sources.
+
+The hashmaps can specify the  "modified" and "created" fields as strings in YYYYMMDDHHMMSSMMM format or as JavaScript date objects. The "tags" field can be given as a JavaScript array of strings or as a TiddlyWiki quoted string (eg, "one [[two three]]").
+
+#### tiddler.fields
+
+Returns a hashmap of tiddler fields, which can be used for read-only access
+
+#### tiddler.hasTag(tag)
+
+Returns a Boolean indicating whether the tiddler has a particular tag.
+
+#### tiddler.cache(name[,value])
+
+Returns or sets the value of a named cache object associated with the tiddler.
+
+### TiddlerConverters.js
+
+This class acts as a factory for tiddler serializers and deserializers.
+
+#### var tiddlerConverters = new TiddlerConverters()
+
+Creates a tiddler converter factory
+
+#### tiddlerConverters.registerSerializer(extension,mimeType,serializer)
+
+Registers a function that knows how to serialise a tiddler into a representation identified by a file extension and a MIME type. The serializer is called with a tiddler to convert and returns the string representation:
+
+	Tiddler.registerSerializer(".tiddler","application/x-tiddler-html-div",function (tiddler) {
+		return "<div" + "...>" + "</div>"; 
+	});
+
+#### tiddlerConverters.registerDeserializer(extension,mimeType,deserializer)
+
+Registers a function that knows how to deserialize one or more tiddlers from a block of text identified by a particular file extension and a MIME type. The deserializer is called with the text to convert and should return an array of tiddler field hashmaps:
+
+	Tiddler.registerDeserializer(".tid","application/x-tiddler",function (text,srcFields) {
+		var fields = copy(SrcFields);
+		// Assemble the fields from the text
+		return [fields];
+	});
+
+#### tiddlerConverters.deserialize(type,text,srcFields)
+
+Given a block of text and a MIME type or file extension, returns an array of hashmaps of tiddler fields. One or more source fields can be provided to pre-populate the tiddler before the text is parsed.
+
+If the type is not recognised then the raw text is assigned to the `text` field.
+
+#### tiddlerConverters.serialize(tiddler,type)
+
+Serializes a tiddler into a text representation identified by a MIME type or file extension.
+
+For example:
+
+	console.log(tiddlerConverters.serialize(tiddler,".tid"));
+
+### TiddlerInput.js and TiddlerOutput.js
+
+Contain classes that can be registered with a TiddlerConverters object to common formats.
+
+#### TiddlerInput.register(tiddlerConverters)
+
+Registers deserializers for these input types:
+
+	Extension	MIME types						Description
+	---------	---------						-----------
+	.tiddler	application/x-tiddler-html-div	TiddlyWiki storeArea-style <div>
+	.tid		application/x-tiddler			TiddlyWeb-style tiddler text file
+	.txt		text/plain						plain text file interpreted as the tiddler text
+	.html		text/html						plain HTML file interpreted as the tiddler text
+	.js			application/javascript			JavaScript file interpreted as the tiddler text
+	.json		application/json				JSON object containing an array of tiddler field hashmaps
+	.tiddlywiki	application/x-tiddlywiki		TiddlyWiki HTML document containing one or more tiddler <div>s
+
+#### TiddlerOutput.register(tiddlerConverters)
+
+Registers serializers for these output types:
+
+	Extension	MIME types						Description
+	---------	---------						-----------
+	.tiddler	application/x-tiddler-html-div	TiddlyWiki storeArea-style <div>
+	.tid		application/x-tiddler			TiddlyWeb-style tiddler text file
+
+### TextProcessors.js
+
+Text processors are components that know how to parse and render tiddlers of particular types. The core of TiddlyWiki is the WikiText processor, which can parse TiddlyWiki wikitext into a JavaScript object tree representation, and then render the tree into HTML or plain text. Other text processors planned include:
+
+* `JSONText` to allow JSON objects to display nicely, and make their content available with TiddlyWiki section/slice notation
+* `CSSText` to parse CSS, and process extensions such as transclusion, theming and variables
+* `JavaScriptText` to parse JavaScript tiddlers for clearer display, and allow sandboxed execution through compilation
+
+Note that text processors encapsulate two operations: parsing into a tree, and rendering that tree into text representations. Parsing doesn't need a context, but rendering needs to have access to a context consisting of a WikiStore to use to retrieve any referenced tiddlers, and the title of the tiddler that is being rendered.
+
+#### textProcessors = new TextProcessors()
+
+Applications should create a TextProcessors object to keep track of the available text processors.
+
+#### textProcessors.registerTextProcessor(mimeType,textProcessor)
+
+Registers an instance of a text processor class to handle text with a particular MIME type. For example:
+
+	var options = {
+		...
+	};
+	textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor(options));
+
+The text processor object must support the following methods:
+
+	// Parses some text and returns a parse tree object
+	var parseTree = textProcessor.parse(text)
+
+Parser objects support the following methods:
+
+	// Renders a subnode of the parse tree to a representation identified by MIME type,
+	// as if rendered within the context of the specified WikiStore and tiddler title
+	var renderedText = parseTree.render(type,treenode,store,title)
+
+#### textProcessors.parse(type,text)
+
+Chooses a text processor based on the MIME type of the content and calls the `parse` method to parse the text into a parse tree. Returns null if the type was not recognised by a registered parser.
+
+If the MIME type is unrecognised or unknown, it defaults to "text/x-tiddlywiki".
+
+### WikiTextProcessor.js
+
+A text processor that parses and renders TiddlyWiki style wiki text.
+
+This module privately includes the following modules:
+
+* WikiTextParser.js containing the wiki text parsing engine
+* WikiTextRules.js containing the rules driving the wiki text parsing engine
+* WikiTextRenderer.js containing the wiki text rendering engine
+* WikiTextMacros.js containing the predefined macros used by the renderer
+
+#### var wikiTextProcessor = new WikiTextProcessor(options)
+
+Creates a new instance of the wiki text processor with the specified options. The options are a hashmap of optional members as follows:
+
+* **enableRules:** An array of names of wiki text rules to enable. If not specified, all rules are available
+* **extraRules:** An array of additional rule handlers to add
+* **enableMacros:** An array of names of macros to enable. If not specified, all macros are available
+* **extraMacros:** An array of additional macro handlers to add
+
+### WikiStore.js
+
+A collection of uniquely titled tiddlers. Although the tiddlers themselves are immutable, new tiddlers can be stored under an existing title, replacing the previous tiddler.
+
+Each wiki store is connected to a shadow store that is also a WikiStore() object. Under certain circumstances, when an attempt is made to retrieve a tiddler that doesn't exist in the store, the search continues into its shadow store (and so on, if the shadow store itself has a shadow store).
+
+#### var store = new WikiStore(options)
+
+Creates a new wiki store. The options are a hashmap of optional members as follows:
+
+* **textProcessors:** A reference to the TextProcessors() instance to be used to resolve parsing and rendering requests
+* **shadowStore:** An optional reference to an existing WikiStore to use as the source of shadow tiddlers. Pass null to disable shadow tiddlers for the new store
+
+#### store.shadows
+
+Exposes a reference to the shadow store for this store.
+
+#### store.clear()
+
+Clears the store of all tiddlers.
+
+#### store.getTiddler(title)
+
+Attempts to retrieve the tiddler with a given title. Returns `null` if the tiddler doesn't exist.
+
+#### store.getTiddlerText(title,defaultText)
+
+Retrieves the text of a particular tiddler. If the tiddler doesn't exist, then the defaultText is returned, or `null` if not specified.
+
+#### store.deleteTiddler(title)
+
+Deletes the specified tiddler from the store.
+
+#### store.tiddlerExists(title)
+
+Returns a boolean indicating whether a particular tiddler exists.
+
+#### store.addTiddler(tiddler)
+
+Adds the specified tiddler object to the store. The tiddler can be specified as a Tiddler() object or a hashmap of tiddler fields.
+
+#### store.forEachTiddler([sortField,]callback)
+
+Invokes a callback for each tiddler in the store, optionally sorted by a particular field. The callback is called with the title of the tiddler and a reference to the tiddler itself. For example:
+
+	store.forEachTiddler(function(title,tiddler) {
+		console.log(title);
+	});
+
+#### store.parseTiddler(title)
+
+Returns the parse tree object for a tiddler, which may be cached within the tiddler.
+
+#### store.renderTiddler(type,title)
+
+Returns a dynamically generated rendering of the tiddler in a representation identified by a MIME type.
+
+### Recipe.js
+
+The Recipe() class loads a TiddlyWiki recipe file, resolving references to subrecipe files. Tiddlers referenced by the recipe are loaded into a WikiStore. A fully loaded recipe can then be cooked to produce an HTML or RSS TiddlyWiki representation of the recipe.
+
+#### var recipe = new Recipe(options,callback)
+
+Creates a new Recipe object by loading the specified recipe file. On completion the callback is invoked with a single parameter `err` that is null if the recipe loading was successful, or an Error() object otherwise.
+
+	var recipe = new Recipe({
+		filepath: "recent.recipe",
+		tiddlerConverters: tiddlerConverters,
+		store: store	
+	},function callback(err) {
+		if(err) {
+			throw err;
+		} else {
+			console.log(recipe.cook())
+		}
+	}
+
+Options is a hashmap with the following mandatory fields:
+
+* **filepath:** The filepath to the recipe file to load
+* **tiddlerConverters:** The TiddlerConverters() object to use to serialize and deserialize tiddlers
+* **textProcessors:** The TextProcessors() object to use to parse and render tiddler text
+* **store:** The WikiStore object to use to store the tiddlers in the recipe
+
+The options can also contain these optional fields:
+
+* (none at present)
+
+#### recipe.cook()
+
+Cooks a TiddlyWiki HTML file from the recipe and returns it as a string.
+
+#### recipe.cookRss()
+
+Cooks a TiddlyWiki RSS file from the recipe and returns it as a string.

tiddlywiki.js

@@ -8,9 +8,10 @@ TiddlyWiki command line interface
 var WikiStore = require("./js/WikiStore.js").WikiStore,
 	Tiddler = require("./js/Tiddler.js").Tiddler,
 	Recipe = require("./js/Recipe.js").Recipe,
-	Tiddler = require("./js/Tiddler.js").Tiddler,
 	tiddlerInput = require("./js/TiddlerInput.js"),
 	tiddlerOutput = require("./js/TiddlerOutput.js"),
+	TextProcessors = require("./js/TextProcessors.js").TextProcessors,
+	WikiTextProcessor = require("./js/WikiTextProcessor.js").WikiTextProcessor,
 	TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters,
 	util = require("util"),
 	fs = require("fs"),
@@ -42,19 +43,27 @@ var parseOptions = function(args,defaultSwitch) {
 	return result;
 };
 
-var tiddlerConverters = new TiddlerConverters(),
+var textProcessors = new TextProcessors(),
+	tiddlerConverters = new TiddlerConverters(),
 	switches = parseOptions(Array.prototype.slice.call(process.argv,2),"dummy"),
-	store = new WikiStore(),
+	store = new WikiStore({
+		textProcessors: textProcessors
+	}),
 	recipe = null,
 	lastRecipeFilepath = null,
 	currSwitch = 0;
 
+
+textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor({}));
 // Register the standard tiddler serializers and deserializers
 tiddlerInput.register(tiddlerConverters);
 tiddlerOutput.register(tiddlerConverters);
 
 // Add the shadow tiddlers that are built into TiddlyWiki
-var shadowShadowStore = new WikiStore(null),
+var shadowShadowStore = new WikiStore({
+		textProcessors: textProcessors,
+		shadowStore: null
+	}),
 	shadowShadows = [
 		{title: "StyleSheet", text: ""},
 		{title: "MarkupPreHead", text: ""},
@@ -123,7 +132,8 @@ var commandLineSwitches = {
 				recipe = new Recipe({
 					filepath: args[0],
 					store: store,
-					tiddlerConverters: tiddlerConverters
+					tiddlerConverters: tiddlerConverters,
+					textProcessors: textProcessors
 				},function() {
 					callback(null);
 				});
@@ -203,7 +213,9 @@ var commandLineSwitches = {
 			// Dumbly, this implementation wastes the recipe processing that happened on the --recipe switch
 			http.createServer(function(request, response) {
 				response.writeHead(200, {"Content-Type": "text/html"});
-				store = new WikiStore();
+				store = new WikiStore({
+					textProcessors: textProcessors
+				});
 				recipe = new Recipe(store,lastRecipeFilepath,function() {
 					response.end(recipe.cook(), "utf8");	
 				});
@@ -219,7 +231,7 @@ var commandLineSwitches = {
 					tiddler = store.getTiddler(title);
 				if(tiddler) {
 					response.writeHead(200, {"Content-Type": "text/html"});
-					response.end(tiddler.getParseTree().render("text/html"),"utf8");					
+					response.end(store.renderTiddler("text/html",title),"utf8");					
 				} else {
 					response.writeHead(404);
 					response.end();

wikitest.js

@@ -14,7 +14,8 @@ verifying that the output matches `<tiddlername>.html` and `<tiddlername>.txt`.
 
 var Tiddler = require("./js/Tiddler.js").Tiddler,
 	WikiStore = require("./js/WikiStore.js").WikiStore,
-	WikiTextRenderer = require("./js/WikiTextRenderer.js").WikiTextRenderer,
+	TextProcessors = require("./js/TextProcessors.js").TextProcessors,
+	WikiTextProcessor = require("./js/WikiTextProcessor.js").WikiTextProcessor,
 	TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters,
 	tiddlerInput = require("./js/TiddlerInput.js"),
 	utils = require("./js/Utils.js"),
@@ -23,12 +24,16 @@ var Tiddler = require("./js/Tiddler.js").Tiddler,
 	path = require("path");
 
 var testdirectory = process.argv[2],
+	textProcessors = new TextProcessors(),
 	tiddlerConverters = new TiddlerConverters(),
-	store = new WikiStore(),
+	store = new WikiStore({
+		textProcessors: textProcessors
+	}),
 	files = fs.readdirSync(testdirectory),
 	titles = [],
 	f,t,extname,basename;
 
+textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor({}));
 tiddlerInput.register(tiddlerConverters);
 
 for(f=0; f<files.length; f++) {
@@ -46,10 +51,9 @@ for(f=0; f<files.length; f++) {
 for(t=0; t<titles.length; t++) {
 	var htmlTarget = fs.readFileSync(path.resolve(testdirectory,titles[t] + ".html"),"utf8"),
 		plainTarget = fs.readFileSync(path.resolve(testdirectory,titles[t] + ".txt"),"utf8"),
-		parser = store.getTiddler(titles[t]).getParseTree(),
+		tiddler = store.getTiddler(titles[t]),
-		renderer = new WikiTextRenderer(parser,store,titles[t]),
+		htmlRender = store.renderTiddler("text/html",titles[t]),
-		htmlRender = renderer.render("text/html"),
+		plainRender = store.renderTiddler("text/plain",titles[t]);
-		plainRender = renderer.render("text/plain");
 	if(htmlTarget !== htmlRender) {
 		console.error("Tiddler %s html error\nTarget: %s\nFound: %s\n",titles[t],htmlTarget,htmlRender);
 	}