Integrated the TiddlyWiki wikifier

A large refactoring to tidy up the interface of the TiddlyWiki wikifier
code, and package it as a wiki text parser.

js/Tiddler.js

@@ -1,17 +1,31 @@
 /*
 
 Tiddlers are an immutable dictionary of name:value pairs called fields. Values can be a string, an array
-of strings, or a date.
+of strings, or a date. The only field that is required is the `title` field, but useful tiddlers also
+have a `text` field, and some of the standard fields `modified`, `modifier`, `created`, `creator`,
+`tags` and `type`.
 
 Hardcoded in the system is the knowledge that the 'tags' field is a string array, and that the 'modified'
 and 'created' fields are dates. All other fields are strings.
 
+Tiddler text is parsed into a tree representation. The parsing performed depends on the type of the
+tiddler: wiki text tiddlers are parsed by the wikifier, JSON tiddlers are parsed by JSON.parse(), and so on.
+
+The parse tree representation of the tiddler is then used for general computations involving the tiddler. For
+example, outbound links can be quickly extracted from a parsed tiddler. Parsing doesn't depend on external
+context such as the content of other tiddlers, and so the resulting parse tree can be safely cached.
+
+Rendering a tiddler is the process of producing a representation of the parse tree in the required
+format (typically HTML) - this is done within the context of a TiddlyWiki store object, not at the level of
+individual tiddlers.
+
 The Tiddler object exposes the following API
 
 new Tiddler(src) - create a Tiddler given a hashmap of field values or a tiddler to clone
 new Tiddler(src1,src2) - create a Tiddler with the union of the fields from the
 						sources, with the rightmost taking priority
-Tiddler.fields - hashmap of tiddler fields
+Tiddler.fields - hashmap of tiddler fields, OK for read-only access
+tiddler.getParseTree() - returns the parse tree for the tiddler
 
 The hashmap(s) can specify the  "modified" and "created" fields as strings in YYYYMMDDHHMMSSMMM
 format or as JavaScript date objects. The "tags" field can be given as a JavaScript array of strings or
@@ -23,7 +37,8 @@ as a TiddlyWiki quoted string (eg, "one [[two three]]").
 "use strict";
 
 var utils = require("./Utils.js"),
-	ArgParser = require("./ArgParser.js").ArgParser;
+	ArgParser = require("./ArgParser.js").ArgParser,
+	WikiTextParser = require("./WikiTextParser.js").WikiTextParser;
 
 var Tiddler = function(/* tiddler,fields */) {
 	this.fields = {};
@@ -36,7 +51,7 @@ var Tiddler = function(/* tiddler,fields */) {
 			src = arg;
 		}
 		for(var t in src) {
-			var f = this.parseField(t,src[t]);
+			var f = this.parseTiddlerField(t,src[t]);
 			if(f !== null) {
 				this.fields[t] = f;
 			}
@@ -44,10 +59,10 @@ var Tiddler = function(/* tiddler,fields */) {
 	}
 };
 
-Tiddler.prototype.parseField = function(name,value) {
+Tiddler.prototype.parseTiddlerField = function(name,value) {
-	var type = Tiddler.specialFields[name];
+	var type = Tiddler.specialTiddlerFields[name];
 	if(type) {
-		return Tiddler.specialParsers[type](value);
+		return Tiddler.specialTiddlerFieldParsers[type](value);
 	} else if (typeof value === "string") {
 		return value;
 	} else {
@@ -56,13 +71,13 @@ Tiddler.prototype.parseField = function(name,value) {
 };
 
 // These are the non-string fields
-Tiddler.specialFields = {
+Tiddler.specialTiddlerFields = {
 	"created": "date",
 	"modified": "date",
 	"tags": "array"
 };
 
-Tiddler.specialParsers = {
+Tiddler.specialTiddlerFieldParsers = {
 	date: function(value) {
 		if(typeof value === "string") {
 			return utils.convertFromYYYYMMDDHHMMSSMMM(value);
@@ -90,4 +105,27 @@ Tiddler.specialParsers = {
 	}
 };
 
+Tiddler.prototype.getParseTree = function() {
+	if(!this.parseTree) {
+		var type = this.fields.type || "application/x-tiddlywikitext",
+		parser = Tiddler.tiddlerTextParsers[type];
+		if(parser) {
+			this.parseTree = Tiddler.tiddlerTextParsers[type].call(this);
+		}
+	}
+	return this.parseTree;
+};
+
+Tiddler.tiddlerTextParsers = {
+	"application/x-tiddlywikitext": function() {
+		return new WikiTextParser(this.fields.text);
+	},
+	"application/javascript": function() {
+		// Would be useful to parse so that we can do syntax highlighting and debugging
+	},
+	"application/json": function() {
+		return JSON.parse(this.fields.text);
+	}
+};
+
 exports.Tiddler = Tiddler;

js/TiddlerOutput.js

@@ -73,9 +73,9 @@ tiddlerOutput.outputTiddlerDiv = function(tid) {
 	outputAttribute("title");
 	outputAttribute("creator");
 	outputAttribute("modifier");
-	outputAttribute("created", function(v) {return utils.convertToYYYYMMDDHHMM(v)});
+	outputAttribute("created", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
-	outputAttribute("modified", function(v) {return utils.convertToYYYYMMDDHHMM(v)});
+	outputAttribute("modified", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
-	outputAttribute("tags", function(v) {return tiddlerOutput.stringifyTags(v)});
+	outputAttribute("tags", function(v) {return tiddlerOutput.stringifyTags(v);});
 	// Output any other attributes
 	for(t in attributes) {
 		outputAttribute(t,null,true);

js/TiddlyWiki.js

@@ -1,7 +1,8 @@
-/*global require: false, exports: false */
+/*global require: false, exports: false, console: false */
 "use strict";
 
-var Tiddler = require("./Tiddler.js").Tiddler;
+var Tiddler = require("./Tiddler.js").Tiddler,
+	util = require("util");
 
 var TiddlyWiki = function TiddlyWiki(shadowStore) {
 	this.tiddlers = {};

js/WikiTextParser.js

@@ -0,0 +1,153 @@
+/*
+
+WikiTextParser.js
+
+Parses a block of tiddlywiki-format wiki text into a parse tree object.
+
+HTML elements are stored in the tree like this:
+
+	{type: "div", attributes: {
+			attr1: value,
+			style: {
+				name: value,
+				name2: value2
+			}
+		}, children: [
+			{child},
+			{child},		
+		]}
+
+Text nodes are:
+
+	{type: "text", value: "string of text node"}
+
+*/
+
+/*global require: false, exports: false */
+"use strict";
+
+var Tiddler = require("./Tiddler.js").Tiddler,
+	wikiTextRules = require("./WikiTextRules.js").wikiTextRules,
+	utils = require("./Utils.js"),
+	util = require("util");
+
+var WikiTextParser = function(text) {
+	this.autoLinkWikiWords = true;
+	this.source = text;
+	this.nextMatch = 0;
+	this.tree = [];
+	this.output = null;
+	this.subWikify(this.tree);
+};
+
+WikiTextParser.prototype.outputText = function(place,startPos,endPos)
+{
+	if(startPos < endPos) {
+		place.push({type: "text", value: this.source.substring(startPos,endPos)});
+	}
+};
+
+WikiTextParser.prototype.subWikify = function(output,terminator)
+{
+	// Handle the terminated and unterminated cases separately, this speeds up wikifikation by about 30%
+	if(terminator)
+		this.subWikifyTerm(output,new RegExp("(" + terminator + ")","mg"));
+	else
+		this.subWikifyUnterm(output);
+};
+
+WikiTextParser.prototype.subWikifyUnterm = function(output)
+{
+	// subWikify can be indirectly recursive, so we need to save the old output pointer
+	var oldOutput = this.output;
+	this.output = output;
+	// Get the first match
+	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	var ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+	while(ruleMatch) {
+		// Output any text before the match
+		if(ruleMatch.index > this.nextMatch)
+			this.outputText(this.output,this.nextMatch,ruleMatch.index);
+		// Set the match parameters for the handler
+		this.matchStart = ruleMatch.index;
+		this.matchLength = ruleMatch[0].length;
+		this.matchText = ruleMatch[0];
+		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		// Figure out which rule matched and call its handler
+		var t;
+		for(t=1; t<ruleMatch.length; t++) {
+			if(ruleMatch[t]) {
+				wikiTextRules.rules[t-1].handler(this);
+				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				break;
+			}
+		}
+		// Get the next match
+		ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+	}
+	// Output any text after the last match
+	if(this.nextMatch < this.source.length) {
+		this.outputText(this.output,this.nextMatch,this.source.length);
+		this.nextMatch = this.source.length;
+	}
+	// Restore the output pointer
+	this.output = oldOutput;
+};
+
+WikiTextParser.prototype.subWikifyTerm = function(output,terminatorRegExp)
+{
+	// subWikify can be indirectly recursive, so we need to save the old output pointer
+	var oldOutput = this.output;
+	this.output = output;
+	// Get the first matches for the rule and terminator RegExps
+	terminatorRegExp.lastIndex = this.nextMatch;
+	var terminatorMatch = terminatorRegExp.exec(this.source);
+	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	var ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+	while(terminatorMatch || ruleMatch) {
+		// Check for a terminator match before the next rule match
+		if(terminatorMatch && (!ruleMatch || terminatorMatch.index <= ruleMatch.index)) {
+			// Output any text before the match
+			if(terminatorMatch.index > this.nextMatch)
+				this.outputText(this.output,this.nextMatch,terminatorMatch.index);
+			// Set the match parameters
+			this.matchText = terminatorMatch[1];
+			this.matchLength = terminatorMatch[1].length;
+			this.matchStart = terminatorMatch.index;
+			this.nextMatch = this.matchStart + this.matchLength;
+			// Restore the output pointer
+			this.output = oldOutput;
+			return;
+		}
+		// It must be a rule match; output any text before the match
+		if(ruleMatch.index > this.nextMatch)
+			this.outputText(this.output,this.nextMatch,ruleMatch.index);
+		// Set the match parameters
+		this.matchStart = ruleMatch.index;
+		this.matchLength = ruleMatch[0].length;
+		this.matchText = ruleMatch[0];
+		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		// Figure out which rule matched and call its handler
+		var t;
+		for(t=1; t<ruleMatch.length; t++) {
+			if(ruleMatch[t]) {
+				wikiTextRules.rules[t-1].handler(this);
+				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				break;
+			}
+		}
+		// Get the next match
+		terminatorRegExp.lastIndex = this.nextMatch;
+		terminatorMatch = terminatorRegExp.exec(this.source);
+		ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+	}
+	// Output any text after the last match
+	if(this.nextMatch < this.source.length) {
+		this.outputText(this.output,this.nextMatch,this.source.length);
+		this.nextMatch = this.source.length;
+	}
+	// Restore the output pointer
+	this.output = oldOutput;
+};
+
+exports.WikiTextParser = WikiTextParser;

js/WikiTextRules.js

@@ -1,10 +1,7 @@
 /*global require: false, exports: false, process: false */
 "use strict";
 
-var Tiddler = require("./Tiddler.js").Tiddler,
+var util = require("util");
-	TiddlyWiki = require("./TiddlyWiki.js").TiddlyWiki,
-	utils = require("./Utils.js"),
-	util = require("util");
 
 var textPrimitives = {
 	upperLetter: "[A-Z\u00c0-\u00de\u0150\u0170]",
@@ -40,30 +37,30 @@ textPrimitives.tiddlerAnyLinkRegExp = new RegExp("("+ textPrimitives.wikiLink +
 	textPrimitives.brackettedLink + ")|(?:" +
 	textPrimitives.urlPattern + ")","mg");
 
-function Formatter()
+function WikiTextRules()
 {
 	var pattern = [];
-	this.formatters = Formatter.formatters;
+	this.rules = WikiTextRules.rules;
-	for(var n=0; n<this.formatters.length; n++) {
+	for(var n=0; n<this.rules.length; n++) {
-		pattern.push("(" + this.formatters[n].match + ")");
+		pattern.push("(" + this.rules[n].match + ")");
 	}
-	this.formatterRegExp = new RegExp(pattern.join("|"),"mg");
+	this.rulesRegExp = new RegExp(pattern.join("|"),"mg");
 }
 
-Formatter.createElementAndWikify = function(w) {
+WikiTextRules.createElementAndWikify = function(w) {
 	var e = {type: this.element, children: []};
 	w.output.push(e);
 	w.subWikifyTerm(e.children,this.termRegExp);
 };
 
-Formatter.setAttr = function(e,attr,value) {
+WikiTextRules.setAttr = function(e,attr,value) {
 	if(!"attributes" in e) {
 		e.attributes = {};
 	}
 	e.attributes[attr] = value;
-}
+};
 
-Formatter.inlineCssHelper = function(w) {
+WikiTextRules.inlineCssHelper = function(w) {
 	var styles = [];
 	textPrimitives.cssLookaheadRegExp.lastIndex = w.nextMatch;
 	var lookaheadMatch = textPrimitives.cssLookaheadRegExp.exec(w.source);
@@ -88,7 +85,7 @@ Formatter.inlineCssHelper = function(w) {
 	return styles;
 };
 
-Formatter.applyCssHelper = function(e,styles) {
+WikiTextRules.applyCssHelper = function(e,styles) {
 	if(!"attributes" in e) {
 		e.attributes = {};
 	}
@@ -100,7 +97,7 @@ Formatter.applyCssHelper = function(e,styles) {
 	}
 };
 
-Formatter.enclosedTextHelper = function(w) {
+WikiTextRules.enclosedTextHelper = function(w) {
 	this.lookaheadRegExp.lastIndex = w.matchStart;
 	var lookaheadMatch = this.lookaheadRegExp.exec(w.source);
 	if(lookaheadMatch && lookaheadMatch.index == w.matchStart) {
@@ -112,11 +109,7 @@ Formatter.enclosedTextHelper = function(w) {
 	}
 };
 
-Formatter.isExternalLink = function(w,link) {
+WikiTextRules.isExternalLink = function(w,link) {
-	if(w.store.tiddlerExists(link) || w.store.isShadowTiddler(link)) {
-		// Definitely not an external link
-		return false;
-	}
 	var urlRegExp = new RegExp(textPrimitives.urlPattern,"mg");
 	if(urlRegExp.exec(link)) {
 		// Definitely an external link
@@ -130,7 +123,7 @@ Formatter.isExternalLink = function(w,link) {
 	return false;
 };
 
-Formatter.formatters = [
+WikiTextRules.rules = [
 {
 	name: "table",
 	match: "^\\|(?:[^\\n]*)\\|(?:[fhck]?)$",
@@ -217,7 +210,7 @@ Formatter.formatters = [
 			} else {
 				// Cell
 				w.nextMatch++;
-				var styles = Formatter.inlineCssHelper(w);
+				var styles = WikiTextRules.inlineCssHelper(w);
 				var spaceLeft = false;
 				var chr = w.source.substr(w.nextMatch,1);
 				while(chr == " ") {
@@ -240,7 +233,7 @@ Formatter.formatters = [
 					cell.attributes.colspan = colSpanCount;
 					colSpanCount = 1;
 				}
-				Formatter.applyCssHelper(cell,styles);
+				WikiTextRules.applyCssHelper(cell,styles);
 				w.subWikifyTerm(cell,this.cellTermRegExp);
 				if(w.matchText.substr(w.matchText.length-2,1) == " ") // spaceRight
 					cell.attributes.align = spaceLeft ? "center" : "left";
@@ -334,7 +327,7 @@ Formatter.formatters = [
 	match: "^<<<\\n",
 	termRegExp: /(^<<<(\n|$))/mg,
 	element: "blockquote",
-	handler: Formatter.createElementAndWikify
+	handler: WikiTextRules.createElementAndWikify
 },
 
 {
@@ -405,7 +398,7 @@ Formatter.formatters = [
 		default:
 			break;
 		}
-		Formatter.enclosedTextHelper.call(this,w);
+		WikiTextRules.enclosedTextHelper.call(this,w);
 	}
 },
 
@@ -448,7 +441,7 @@ Formatter.formatters = [
 			if(lookaheadMatch[3]) {
 				// Pretty bracketted link
 				var link = lookaheadMatch[3];
-				if(!lookaheadMatch[2] && Formatter.isExternalLink(w,link)) {
+				if(!lookaheadMatch[2] && WikiTextRules.isExternalLink(w,link)) {
 					e = {type: "a", href: link, children: []};
 				} else {
 					e = {type: "tiddlerLink", href: link, children: []};
@@ -482,7 +475,7 @@ Formatter.formatters = [
 				return;
 			}
 		}
-		if(w.autoLinkWikiWords || w.store.isShadowTiddler(w.matchText)) {
+		if(w.autoLinkWikiWords) {
 			var link = {type: "tiddlerLink", href: w.matchText, children: []};
 			w.output.push(link);
 			w.outputText(link.children,w.matchStart,w.nextMatch);
@@ -516,7 +509,7 @@ Formatter.formatters = [
 			var e = w.output;
 			if(lookaheadMatch[5]) {
 				var link = lookaheadMatch[5],t;
-				if(Formatter.isExternalLink(w,link)) {
+				if(WikiTextRules.isExternalLink(w,link)) {
 					t = {type: "a", href: link, children: []};
 					w.output.push(t);
 					e = t.children;
@@ -530,12 +523,12 @@ Formatter.formatters = [
 			var img = {type: "img"};
 			e.push(img);
 			if(lookaheadMatch[1])
-				Formatter.setAttr(img,"align","left");
+				WikiTextRules.setAttr(img,"align","left");
 			else if(lookaheadMatch[2])
-				Formatter.setAttr(img,"align","right");
+				WikiTextRules.setAttr(img,"align","right");
 			if(lookaheadMatch[3]) {
-				Formatter.setAttr(img,"title",lookaheadMatch[3]);
+				WikiTextRules.setAttr(img,"title",lookaheadMatch[3]);
-				Formatter.setAttr(img,"alt",lookaheadMatch[3]);
+				WikiTextRules.setAttr(img,"alt",lookaheadMatch[3]);
 			}
 			img.src = lookaheadMatch[4];
 			w.nextMatch = this.lookaheadRegExp.lastIndex;
@@ -632,11 +625,11 @@ Formatter.formatters = [
 		case "@@":
 			var e = {type: "span", attributes: {}, children: []};
 			w.output.push(e);
-			var styles = Formatter.inlineCssHelper(w);
+			var styles = WikiTextRules.inlineCssHelper(w);
 			if(styles.length === 0)
 				e.className = "marked";
 			else
-				Formatter.applyCssHelper(e,styles);
+				WikiTextRules.applyCssHelper(e,styles);
 			w.subWikifyTerm(e.children,/(@@)/mg);
 			break;
 		case "{{":
@@ -700,4 +693,4 @@ Formatter.formatters = [
 
 ];
 
-exports.Formatter = Formatter;
+exports.wikiTextRules = new WikiTextRules();

js/Wikifier.js

@@ -1,185 +0,0 @@
-/*
-
-Wikifier for TiddlyWiki format text
-
-The wikifier parses wikitext into an intermediate tree from which the HTML is generated.
-
-HTML elements are stored in the tree like this:
-
-	{type: "div", attributes: {
-			attr1: value,
-			style: {
-				name: value,
-				name2: value2
-			}
-		}, children: [
-			{child},
-			{child},		
-		]}
-
-Text nodes are:
-
-	{type: "text", value: "string of text node"}
-
-*/
-
-/*global require: false, exports: false, process: false */
-"use strict";
-
-var Tiddler = require("./Tiddler.js").Tiddler,
-	TiddlyWiki = require("./TiddlyWiki.js").TiddlyWiki,
-	utils = require("./Utils.js"),
-	util = require("util");
-
-// Construct a wikifier object around a Formatter() object
-var Wikifier = function(store,formatter) {
-	this.store = store;
-	this.formatter = formatter;
-	this.autoLinkWikiWords = true;
-};
-
-// Wikify a string as if it were from a particular tiddler and return it as an HTML string
-Wikifier.prototype.wikify = function(source,tiddler) {
-	this.source = source;
-	this.nextMatch = 0;
-	this.tiddler = tiddler;
-	this.tree = [];
-	this.output = null;
-	this.subWikify(this.tree);
-	return this.tree; // Just return the tree for now
-};
-
-// Wikify a string as if it were from a particular tiddler and return it as plain text
-Wikifier.prototype.wikifyPlain = function(source,tiddler) {
-	this.source = source;
-	this.nextMatch = 0;
-	this.tiddler = tiddler;
-	this.tree = [];
-	this.output = null;
-	this.subWikify(this.tree);
-	var resultText = [],
-		extractText = function(tree) {
-			for(var t=0; t<tree.length; t++) {
-				var node = tree[t];
-				if(node.type === "text") {
-					resultText.push(node.value);
-				} else if(node.children) {
-					extractText(node.children);
-				}
-			}
-		};
-	extractText(this.tree);
-	return resultText.join("");
-};
-
-Wikifier.prototype.outputText = function(place,startPos,endPos)
-{
-	if(startPos < endPos) {
-		place.push({type: "text", value: this.source.substring(startPos,endPos)});
-	}
-};
-
-Wikifier.prototype.subWikify = function(output,terminator)
-{
-	// Handle the terminated and unterminated cases separately, this speeds up wikifikation by about 30%
-	if(terminator)
-		this.subWikifyTerm(output,new RegExp("(" + terminator + ")","mg"));
-	else
-		this.subWikifyUnterm(output);
-};
-
-Wikifier.prototype.subWikifyUnterm = function(output)
-{
-	// subWikify can be indirectly recursive, so we need to save the old output pointer
-	var oldOutput = this.output;
-	this.output = output;
-	// Get the first match
-	this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-	var formatterMatch = this.formatter.formatterRegExp.exec(this.source);
-	while(formatterMatch) {
-		// Output any text before the match
-		if(formatterMatch.index > this.nextMatch)
-			this.outputText(this.output,this.nextMatch,formatterMatch.index);
-		// Set the match parameters for the handler
-		this.matchStart = formatterMatch.index;
-		this.matchLength = formatterMatch[0].length;
-		this.matchText = formatterMatch[0];
-		this.nextMatch = this.formatter.formatterRegExp.lastIndex;
-		// Figure out which formatter matched and call its handler
-		var t;
-		for(t=1; t<formatterMatch.length; t++) {
-			if(formatterMatch[t]) {
-				this.formatter.formatters[t-1].handler(this);
-				this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-				break;
-			}
-		}
-		// Get the next match
-		formatterMatch = this.formatter.formatterRegExp.exec(this.source);
-	}
-	// Output any text after the last match
-	if(this.nextMatch < this.source.length) {
-		this.outputText(this.output,this.nextMatch,this.source.length);
-		this.nextMatch = this.source.length;
-	}
-	// Restore the output pointer
-	this.output = oldOutput;
-};
-
-Wikifier.prototype.subWikifyTerm = function(output,terminatorRegExp)
-{
-	// subWikify can be indirectly recursive, so we need to save the old output pointer
-	var oldOutput = this.output;
-	this.output = output;
-	// Get the first matches for the formatter and terminator RegExps
-	terminatorRegExp.lastIndex = this.nextMatch;
-	var terminatorMatch = terminatorRegExp.exec(this.source);
-	this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-	var formatterMatch = this.formatter.formatterRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
-	while(terminatorMatch || formatterMatch) {
-		// Check for a terminator match before the next formatter match
-		if(terminatorMatch && (!formatterMatch || terminatorMatch.index <= formatterMatch.index)) {
-			// Output any text before the match
-			if(terminatorMatch.index > this.nextMatch)
-				this.outputText(this.output,this.nextMatch,terminatorMatch.index);
-			// Set the match parameters
-			this.matchText = terminatorMatch[1];
-			this.matchLength = terminatorMatch[1].length;
-			this.matchStart = terminatorMatch.index;
-			this.nextMatch = this.matchStart + this.matchLength;
-			// Restore the output pointer
-			this.output = oldOutput;
-			return;
-		}
-		// It must be a formatter match; output any text before the match
-		if(formatterMatch.index > this.nextMatch)
-			this.outputText(this.output,this.nextMatch,formatterMatch.index);
-		// Set the match parameters
-		this.matchStart = formatterMatch.index;
-		this.matchLength = formatterMatch[0].length;
-		this.matchText = formatterMatch[0];
-		this.nextMatch = this.formatter.formatterRegExp.lastIndex;
-		// Figure out which formatter matched and call its handler
-		var t;
-		for(t=1; t<formatterMatch.length; t++) {
-			if(formatterMatch[t]) {
-				this.formatter.formatters[t-1].handler(this);
-				this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-				break;
-			}
-		}
-		// Get the next match
-		terminatorRegExp.lastIndex = this.nextMatch;
-		terminatorMatch = terminatorRegExp.exec(this.source);
-		formatterMatch = this.formatter.formatterRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
-	}
-	// Output any text after the last match
-	if(this.nextMatch < this.source.length) {
-		this.outputText(this.output,this.nextMatch,this.source.length);
-		this.nextMatch = this.source.length;
-	}
-	// Restore the output pointer
-	this.output = oldOutput;
-};
-
-exports.Wikifier = Wikifier;

wikitest.js

@@ -5,27 +5,23 @@ Wikifier test rig
 
 var Tiddler = require("./js/Tiddler.js").Tiddler,
 	TiddlyWiki = require("./js/TiddlyWiki.js").TiddlyWiki,
-	Formatter = require("./js/Formatter.js").Formatter,
-	Wikifier = require("./js/Wikifier.js").Wikifier,
 	utils = require("./js/Utils.js"),
 	util = require("util");
 
 var wikiTest = function(spec) {
 	var t,
 		store = new TiddlyWiki(),
-		formatter = new Formatter(),
-		wikifier = new Wikifier(store,formatter),
 		w;
 	for(t=0; t<spec.tiddlers.length; t++) {
 		store.addTiddler(new Tiddler(spec.tiddlers[t]));
 	}
 	for(t=0; t<spec.tests.length; t++) {
-		w = wikifier.wikify(store.getTiddlerText(spec.tests[t].tiddler));
+		w = store.getTiddler(spec.tests[t].tiddler).getParseTree().tree;
 		if(JSON.stringify(w) !== JSON.stringify(spec.tests[t].output)) {
 			console.error("Failed at tiddler: " + spec.tests[t].tiddler + " with JSON:\n" + util.inspect(w,false,8));
 		}
 	}
-}
+};
 
 wikiTest({ tiddlers: 
    [ { title: 'FirstTiddler',