From 620add5579384ddf6c655b6157feda84afb83423 Mon Sep 17 00:00:00 2001
From: Jeremy Ruston <jeremy@osmosoft.com>
Date: Mon, 5 Dec 2011 16:50:25 +0000
Subject: [PATCH] Integrated the TiddlyWiki wikifier

A large refactoring to tidy up the interface of the TiddlyWiki wikifier
code, and package it as a wiki text parser.
---
 js/Tiddler.js                         |  56 ++++++--
 js/TiddlerOutput.js                   |   6 +-
 js/TiddlyWiki.js                      |   5 +-
 js/WikiTextParser.js                  | 153 +++++++++++++++++++++
 js/{Formatter.js => WikiTextRules.js} |  63 ++++-----
 js/Wikifier.js                        | 185 --------------------------
 wikitest.js                           |   8 +-
 7 files changed, 236 insertions(+), 240 deletions(-)
 create mode 100644 js/WikiTextParser.js
 rename js/{Formatter.js => WikiTextRules.js} (92%)
 delete mode 100755 js/Wikifier.js

diff --git a/js/Tiddler.js b/js/Tiddler.js
index 6969e4180..d0863864c 100755
--- a/js/Tiddler.js
+++ b/js/Tiddler.js
@@ -1,17 +1,31 @@
 /*
 
 Tiddlers are an immutable dictionary of name:value pairs called fields. Values can be a string, an array
-of strings, or a date.
+of strings, or a date. The only field that is required is the `title` field, but useful tiddlers also
+have a `text` field, and some of the standard fields `modified`, `modifier`, `created`, `creator`,
+`tags` and `type`.
 
 Hardcoded in the system is the knowledge that the 'tags' field is a string array, and that the 'modified'
 and 'created' fields are dates. All other fields are strings.
 
+Tiddler text is parsed into a tree representation. The parsing performed depends on the type of the
+tiddler: wiki text tiddlers are parsed by the wikifier, JSON tiddlers are parsed by JSON.parse(), and so on.
+
+The parse tree representation of the tiddler is then used for general computations involving the tiddler. For
+example, outbound links can be quickly extracted from a parsed tiddler. Parsing doesn't depend on external
+context such as the content of other tiddlers, and so the resulting parse tree can be safely cached.
+
+Rendering a tiddler is the process of producing a representation of the parse tree in the required
+format (typically HTML) - this is done within the context of a TiddlyWiki store object, not at the level of
+individual tiddlers.
+
 The Tiddler object exposes the following API
 
 new Tiddler(src) - create a Tiddler given a hashmap of field values or a tiddler to clone
 new Tiddler(src1,src2) - create a Tiddler with the union of the fields from the
 						sources, with the rightmost taking priority
-Tiddler.fields - hashmap of tiddler fields
+Tiddler.fields - hashmap of tiddler fields, OK for read-only access
+tiddler.getParseTree() - returns the parse tree for the tiddler
 
 The hashmap(s) can specify the  "modified" and "created" fields as strings in YYYYMMDDHHMMSSMMM
 format or as JavaScript date objects. The "tags" field can be given as a JavaScript array of strings or
@@ -23,7 +37,8 @@ as a TiddlyWiki quoted string (eg, "one [[two three]]").
 "use strict";
 
 var utils = require("./Utils.js"),
-	ArgParser = require("./ArgParser.js").ArgParser;
+	ArgParser = require("./ArgParser.js").ArgParser,
+	WikiTextParser = require("./WikiTextParser.js").WikiTextParser;
 
 var Tiddler = function(/* tiddler,fields */) {
 	this.fields = {};
@@ -36,7 +51,7 @@ var Tiddler = function(/* tiddler,fields */) {
 			src = arg;
 		}
 		for(var t in src) {
-			var f = this.parseField(t,src[t]);
+			var f = this.parseTiddlerField(t,src[t]);
 			if(f !== null) {
 				this.fields[t] = f;
 			}
@@ -44,10 +59,10 @@ var Tiddler = function(/* tiddler,fields */) {
 	}
 };
 
-Tiddler.prototype.parseField = function(name,value) {
-	var type = Tiddler.specialFields[name];
+Tiddler.prototype.parseTiddlerField = function(name,value) {
+	var type = Tiddler.specialTiddlerFields[name];
 	if(type) {
-		return Tiddler.specialParsers[type](value);
+		return Tiddler.specialTiddlerFieldParsers[type](value);
 	} else if (typeof value === "string") {
 		return value;
 	} else {
@@ -56,13 +71,13 @@ Tiddler.prototype.parseField = function(name,value) {
 };
 
 // These are the non-string fields
-Tiddler.specialFields = {
+Tiddler.specialTiddlerFields = {
 	"created": "date",
 	"modified": "date",
 	"tags": "array"
 };
 
-Tiddler.specialParsers = {
+Tiddler.specialTiddlerFieldParsers = {
 	date: function(value) {
 		if(typeof value === "string") {
 			return utils.convertFromYYYYMMDDHHMMSSMMM(value);
@@ -90,4 +105,27 @@ Tiddler.specialParsers = {
 	}
 };
 
+Tiddler.prototype.getParseTree = function() {
+	if(!this.parseTree) {
+		var type = this.fields.type || "application/x-tiddlywikitext",
+		parser = Tiddler.tiddlerTextParsers[type];
+		if(parser) {
+			this.parseTree = Tiddler.tiddlerTextParsers[type].call(this);
+		}
+	}
+	return this.parseTree;
+};
+
+Tiddler.tiddlerTextParsers = {
+	"application/x-tiddlywikitext": function() {
+		return new WikiTextParser(this.fields.text);
+	},
+	"application/javascript": function() {
+		// Would be useful to parse so that we can do syntax highlighting and debugging
+	},
+	"application/json": function() {
+		return JSON.parse(this.fields.text);
+	}
+};
+
 exports.Tiddler = Tiddler;
diff --git a/js/TiddlerOutput.js b/js/TiddlerOutput.js
index 73230360a..c8918e9c1 100755
--- a/js/TiddlerOutput.js
+++ b/js/TiddlerOutput.js
@@ -73,9 +73,9 @@ tiddlerOutput.outputTiddlerDiv = function(tid) {
 	outputAttribute("title");
 	outputAttribute("creator");
 	outputAttribute("modifier");
-	outputAttribute("created", function(v) {return utils.convertToYYYYMMDDHHMM(v)});
-	outputAttribute("modified", function(v) {return utils.convertToYYYYMMDDHHMM(v)});
-	outputAttribute("tags", function(v) {return tiddlerOutput.stringifyTags(v)});
+	outputAttribute("created", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
+	outputAttribute("modified", function(v) {return utils.convertToYYYYMMDDHHMM(v);});
+	outputAttribute("tags", function(v) {return tiddlerOutput.stringifyTags(v);});
 	// Output any other attributes
 	for(t in attributes) {
 		outputAttribute(t,null,true);
diff --git a/js/TiddlyWiki.js b/js/TiddlyWiki.js
index 677bff938..d2f013bbc 100755
--- a/js/TiddlyWiki.js
+++ b/js/TiddlyWiki.js
@@ -1,7 +1,8 @@
-/*global require: false, exports: false */
+/*global require: false, exports: false, console: false */
 "use strict";
 
-var Tiddler = require("./Tiddler.js").Tiddler;
+var Tiddler = require("./Tiddler.js").Tiddler,
+	util = require("util");
 
 var TiddlyWiki = function TiddlyWiki(shadowStore) {
 	this.tiddlers = {};
diff --git a/js/WikiTextParser.js b/js/WikiTextParser.js
new file mode 100644
index 000000000..0b8753045
--- /dev/null
+++ b/js/WikiTextParser.js
@@ -0,0 +1,153 @@
+/*
+
+WikiTextParser.js
+
+Parses a block of tiddlywiki-format wiki text into a parse tree object.
+
+HTML elements are stored in the tree like this:
+
+	{type: "div", attributes: {
+			attr1: value,
+			style: {
+				name: value,
+				name2: value2
+			}
+		}, children: [
+			{child},
+			{child},		
+		]}
+
+Text nodes are:
+
+	{type: "text", value: "string of text node"}
+
+*/
+
+/*global require: false, exports: false */
+"use strict";
+
+var Tiddler = require("./Tiddler.js").Tiddler,
+	wikiTextRules = require("./WikiTextRules.js").wikiTextRules,
+	utils = require("./Utils.js"),
+	util = require("util");
+
+var WikiTextParser = function(text) {
+	this.autoLinkWikiWords = true;
+	this.source = text;
+	this.nextMatch = 0;
+	this.tree = [];
+	this.output = null;
+	this.subWikify(this.tree);
+};
+
+WikiTextParser.prototype.outputText = function(place,startPos,endPos)
+{
+	if(startPos < endPos) {
+		place.push({type: "text", value: this.source.substring(startPos,endPos)});
+	}
+};
+
+WikiTextParser.prototype.subWikify = function(output,terminator)
+{
+	// Handle the terminated and unterminated cases separately, this speeds up wikifikation by about 30%
+	if(terminator)
+		this.subWikifyTerm(output,new RegExp("(" + terminator + ")","mg"));
+	else
+		this.subWikifyUnterm(output);
+};
+
+WikiTextParser.prototype.subWikifyUnterm = function(output)
+{
+	// subWikify can be indirectly recursive, so we need to save the old output pointer
+	var oldOutput = this.output;
+	this.output = output;
+	// Get the first match
+	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	var ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+	while(ruleMatch) {
+		// Output any text before the match
+		if(ruleMatch.index > this.nextMatch)
+			this.outputText(this.output,this.nextMatch,ruleMatch.index);
+		// Set the match parameters for the handler
+		this.matchStart = ruleMatch.index;
+		this.matchLength = ruleMatch[0].length;
+		this.matchText = ruleMatch[0];
+		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		// Figure out which rule matched and call its handler
+		var t;
+		for(t=1; t<ruleMatch.length; t++) {
+			if(ruleMatch[t]) {
+				wikiTextRules.rules[t-1].handler(this);
+				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				break;
+			}
+		}
+		// Get the next match
+		ruleMatch = wikiTextRules.rulesRegExp.exec(this.source);
+	}
+	// Output any text after the last match
+	if(this.nextMatch < this.source.length) {
+		this.outputText(this.output,this.nextMatch,this.source.length);
+		this.nextMatch = this.source.length;
+	}
+	// Restore the output pointer
+	this.output = oldOutput;
+};
+
+WikiTextParser.prototype.subWikifyTerm = function(output,terminatorRegExp)
+{
+	// subWikify can be indirectly recursive, so we need to save the old output pointer
+	var oldOutput = this.output;
+	this.output = output;
+	// Get the first matches for the rule and terminator RegExps
+	terminatorRegExp.lastIndex = this.nextMatch;
+	var terminatorMatch = terminatorRegExp.exec(this.source);
+	wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+	var ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+	while(terminatorMatch || ruleMatch) {
+		// Check for a terminator match before the next rule match
+		if(terminatorMatch && (!ruleMatch || terminatorMatch.index <= ruleMatch.index)) {
+			// Output any text before the match
+			if(terminatorMatch.index > this.nextMatch)
+				this.outputText(this.output,this.nextMatch,terminatorMatch.index);
+			// Set the match parameters
+			this.matchText = terminatorMatch[1];
+			this.matchLength = terminatorMatch[1].length;
+			this.matchStart = terminatorMatch.index;
+			this.nextMatch = this.matchStart + this.matchLength;
+			// Restore the output pointer
+			this.output = oldOutput;
+			return;
+		}
+		// It must be a rule match; output any text before the match
+		if(ruleMatch.index > this.nextMatch)
+			this.outputText(this.output,this.nextMatch,ruleMatch.index);
+		// Set the match parameters
+		this.matchStart = ruleMatch.index;
+		this.matchLength = ruleMatch[0].length;
+		this.matchText = ruleMatch[0];
+		this.nextMatch = wikiTextRules.rulesRegExp.lastIndex;
+		// Figure out which rule matched and call its handler
+		var t;
+		for(t=1; t<ruleMatch.length; t++) {
+			if(ruleMatch[t]) {
+				wikiTextRules.rules[t-1].handler(this);
+				wikiTextRules.rulesRegExp.lastIndex = this.nextMatch;
+				break;
+			}
+		}
+		// Get the next match
+		terminatorRegExp.lastIndex = this.nextMatch;
+		terminatorMatch = terminatorRegExp.exec(this.source);
+		ruleMatch = wikiTextRules.rulesRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
+	}
+	// Output any text after the last match
+	if(this.nextMatch < this.source.length) {
+		this.outputText(this.output,this.nextMatch,this.source.length);
+		this.nextMatch = this.source.length;
+	}
+	// Restore the output pointer
+	this.output = oldOutput;
+};
+
+exports.WikiTextParser = WikiTextParser;
diff --git a/js/Formatter.js b/js/WikiTextRules.js
similarity index 92%
rename from js/Formatter.js
rename to js/WikiTextRules.js
index e15beee0a..8203800d9 100755
--- a/js/Formatter.js
+++ b/js/WikiTextRules.js
@@ -1,10 +1,7 @@
 /*global require: false, exports: false, process: false */
 "use strict";
 
-var Tiddler = require("./Tiddler.js").Tiddler,
-	TiddlyWiki = require("./TiddlyWiki.js").TiddlyWiki,
-	utils = require("./Utils.js"),
-	util = require("util");
+var util = require("util");
 
 var textPrimitives = {
 	upperLetter: "[A-Z\u00c0-\u00de\u0150\u0170]",
@@ -40,30 +37,30 @@ textPrimitives.tiddlerAnyLinkRegExp = new RegExp("("+ textPrimitives.wikiLink +
 	textPrimitives.brackettedLink + ")|(?:" +
 	textPrimitives.urlPattern + ")","mg");
 
-function Formatter()
+function WikiTextRules()
 {
 	var pattern = [];
-	this.formatters = Formatter.formatters;
-	for(var n=0; n<this.formatters.length; n++) {
-		pattern.push("(" + this.formatters[n].match + ")");
+	this.rules = WikiTextRules.rules;
+	for(var n=0; n<this.rules.length; n++) {
+		pattern.push("(" + this.rules[n].match + ")");
 	}
-	this.formatterRegExp = new RegExp(pattern.join("|"),"mg");
+	this.rulesRegExp = new RegExp(pattern.join("|"),"mg");
 }
 
-Formatter.createElementAndWikify = function(w) {
+WikiTextRules.createElementAndWikify = function(w) {
 	var e = {type: this.element, children: []};
 	w.output.push(e);
 	w.subWikifyTerm(e.children,this.termRegExp);
 };
 
-Formatter.setAttr = function(e,attr,value) {
+WikiTextRules.setAttr = function(e,attr,value) {
 	if(!"attributes" in e) {
 		e.attributes = {};
 	}
 	e.attributes[attr] = value;
-}
+};
 
-Formatter.inlineCssHelper = function(w) {
+WikiTextRules.inlineCssHelper = function(w) {
 	var styles = [];
 	textPrimitives.cssLookaheadRegExp.lastIndex = w.nextMatch;
 	var lookaheadMatch = textPrimitives.cssLookaheadRegExp.exec(w.source);
@@ -88,7 +85,7 @@ Formatter.inlineCssHelper = function(w) {
 	return styles;
 };
 
-Formatter.applyCssHelper = function(e,styles) {
+WikiTextRules.applyCssHelper = function(e,styles) {
 	if(!"attributes" in e) {
 		e.attributes = {};
 	}
@@ -100,7 +97,7 @@ Formatter.applyCssHelper = function(e,styles) {
 	}
 };
 
-Formatter.enclosedTextHelper = function(w) {
+WikiTextRules.enclosedTextHelper = function(w) {
 	this.lookaheadRegExp.lastIndex = w.matchStart;
 	var lookaheadMatch = this.lookaheadRegExp.exec(w.source);
 	if(lookaheadMatch && lookaheadMatch.index == w.matchStart) {
@@ -112,11 +109,7 @@ Formatter.enclosedTextHelper = function(w) {
 	}
 };
 
-Formatter.isExternalLink = function(w,link) {
-	if(w.store.tiddlerExists(link) || w.store.isShadowTiddler(link)) {
-		// Definitely not an external link
-		return false;
-	}
+WikiTextRules.isExternalLink = function(w,link) {
 	var urlRegExp = new RegExp(textPrimitives.urlPattern,"mg");
 	if(urlRegExp.exec(link)) {
 		// Definitely an external link
@@ -130,7 +123,7 @@ Formatter.isExternalLink = function(w,link) {
 	return false;
 };
 
-Formatter.formatters = [
+WikiTextRules.rules = [
 {
 	name: "table",
 	match: "^\\|(?:[^\\n]*)\\|(?:[fhck]?)$",
@@ -217,7 +210,7 @@ Formatter.formatters = [
 			} else {
 				// Cell
 				w.nextMatch++;
-				var styles = Formatter.inlineCssHelper(w);
+				var styles = WikiTextRules.inlineCssHelper(w);
 				var spaceLeft = false;
 				var chr = w.source.substr(w.nextMatch,1);
 				while(chr == " ") {
@@ -240,7 +233,7 @@ Formatter.formatters = [
 					cell.attributes.colspan = colSpanCount;
 					colSpanCount = 1;
 				}
-				Formatter.applyCssHelper(cell,styles);
+				WikiTextRules.applyCssHelper(cell,styles);
 				w.subWikifyTerm(cell,this.cellTermRegExp);
 				if(w.matchText.substr(w.matchText.length-2,1) == " ") // spaceRight
 					cell.attributes.align = spaceLeft ? "center" : "left";
@@ -334,7 +327,7 @@ Formatter.formatters = [
 	match: "^<<<\\n",
 	termRegExp: /(^<<<(\n|$))/mg,
 	element: "blockquote",
-	handler: Formatter.createElementAndWikify
+	handler: WikiTextRules.createElementAndWikify
 },
 
 {
@@ -405,7 +398,7 @@ Formatter.formatters = [
 		default:
 			break;
 		}
-		Formatter.enclosedTextHelper.call(this,w);
+		WikiTextRules.enclosedTextHelper.call(this,w);
 	}
 },
 
@@ -448,7 +441,7 @@ Formatter.formatters = [
 			if(lookaheadMatch[3]) {
 				// Pretty bracketted link
 				var link = lookaheadMatch[3];
-				if(!lookaheadMatch[2] && Formatter.isExternalLink(w,link)) {
+				if(!lookaheadMatch[2] && WikiTextRules.isExternalLink(w,link)) {
 					e = {type: "a", href: link, children: []};
 				} else {
 					e = {type: "tiddlerLink", href: link, children: []};
@@ -482,7 +475,7 @@ Formatter.formatters = [
 				return;
 			}
 		}
-		if(w.autoLinkWikiWords || w.store.isShadowTiddler(w.matchText)) {
+		if(w.autoLinkWikiWords) {
 			var link = {type: "tiddlerLink", href: w.matchText, children: []};
 			w.output.push(link);
 			w.outputText(link.children,w.matchStart,w.nextMatch);
@@ -516,7 +509,7 @@ Formatter.formatters = [
 			var e = w.output;
 			if(lookaheadMatch[5]) {
 				var link = lookaheadMatch[5],t;
-				if(Formatter.isExternalLink(w,link)) {
+				if(WikiTextRules.isExternalLink(w,link)) {
 					t = {type: "a", href: link, children: []};
 					w.output.push(t);
 					e = t.children;
@@ -530,12 +523,12 @@ Formatter.formatters = [
 			var img = {type: "img"};
 			e.push(img);
 			if(lookaheadMatch[1])
-				Formatter.setAttr(img,"align","left");
+				WikiTextRules.setAttr(img,"align","left");
 			else if(lookaheadMatch[2])
-				Formatter.setAttr(img,"align","right");
+				WikiTextRules.setAttr(img,"align","right");
 			if(lookaheadMatch[3]) {
-				Formatter.setAttr(img,"title",lookaheadMatch[3]);
-				Formatter.setAttr(img,"alt",lookaheadMatch[3]);
+				WikiTextRules.setAttr(img,"title",lookaheadMatch[3]);
+				WikiTextRules.setAttr(img,"alt",lookaheadMatch[3]);
 			}
 			img.src = lookaheadMatch[4];
 			w.nextMatch = this.lookaheadRegExp.lastIndex;
@@ -632,11 +625,11 @@ Formatter.formatters = [
 		case "@@":
 			var e = {type: "span", attributes: {}, children: []};
 			w.output.push(e);
-			var styles = Formatter.inlineCssHelper(w);
+			var styles = WikiTextRules.inlineCssHelper(w);
 			if(styles.length === 0)
 				e.className = "marked";
 			else
-				Formatter.applyCssHelper(e,styles);
+				WikiTextRules.applyCssHelper(e,styles);
 			w.subWikifyTerm(e.children,/(@@)/mg);
 			break;
 		case "{{":
@@ -700,4 +693,4 @@ Formatter.formatters = [
 
 ];
 
-exports.Formatter = Formatter;
+exports.wikiTextRules = new WikiTextRules();
diff --git a/js/Wikifier.js b/js/Wikifier.js
deleted file mode 100755
index eda5f523f..000000000
--- a/js/Wikifier.js
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
-
-Wikifier for TiddlyWiki format text
-
-The wikifier parses wikitext into an intermediate tree from which the HTML is generated.
-
-HTML elements are stored in the tree like this:
-
-	{type: "div", attributes: {
-			attr1: value,
-			style: {
-				name: value,
-				name2: value2
-			}
-		}, children: [
-			{child},
-			{child},		
-		]}
-
-Text nodes are:
-
-	{type: "text", value: "string of text node"}
-
-*/
-
-/*global require: false, exports: false, process: false */
-"use strict";
-
-var Tiddler = require("./Tiddler.js").Tiddler,
-	TiddlyWiki = require("./TiddlyWiki.js").TiddlyWiki,
-	utils = require("./Utils.js"),
-	util = require("util");
-
-// Construct a wikifier object around a Formatter() object
-var Wikifier = function(store,formatter) {
-	this.store = store;
-	this.formatter = formatter;
-	this.autoLinkWikiWords = true;
-};
-
-// Wikify a string as if it were from a particular tiddler and return it as an HTML string
-Wikifier.prototype.wikify = function(source,tiddler) {
-	this.source = source;
-	this.nextMatch = 0;
-	this.tiddler = tiddler;
-	this.tree = [];
-	this.output = null;
-	this.subWikify(this.tree);
-	return this.tree; // Just return the tree for now
-};
-
-// Wikify a string as if it were from a particular tiddler and return it as plain text
-Wikifier.prototype.wikifyPlain = function(source,tiddler) {
-	this.source = source;
-	this.nextMatch = 0;
-	this.tiddler = tiddler;
-	this.tree = [];
-	this.output = null;
-	this.subWikify(this.tree);
-	var resultText = [],
-		extractText = function(tree) {
-			for(var t=0; t<tree.length; t++) {
-				var node = tree[t];
-				if(node.type === "text") {
-					resultText.push(node.value);
-				} else if(node.children) {
-					extractText(node.children);
-				}
-			}
-		};
-	extractText(this.tree);
-	return resultText.join("");
-};
-
-Wikifier.prototype.outputText = function(place,startPos,endPos)
-{
-	if(startPos < endPos) {
-		place.push({type: "text", value: this.source.substring(startPos,endPos)});
-	}
-};
-
-Wikifier.prototype.subWikify = function(output,terminator)
-{
-	// Handle the terminated and unterminated cases separately, this speeds up wikifikation by about 30%
-	if(terminator)
-		this.subWikifyTerm(output,new RegExp("(" + terminator + ")","mg"));
-	else
-		this.subWikifyUnterm(output);
-};
-
-Wikifier.prototype.subWikifyUnterm = function(output)
-{
-	// subWikify can be indirectly recursive, so we need to save the old output pointer
-	var oldOutput = this.output;
-	this.output = output;
-	// Get the first match
-	this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-	var formatterMatch = this.formatter.formatterRegExp.exec(this.source);
-	while(formatterMatch) {
-		// Output any text before the match
-		if(formatterMatch.index > this.nextMatch)
-			this.outputText(this.output,this.nextMatch,formatterMatch.index);
-		// Set the match parameters for the handler
-		this.matchStart = formatterMatch.index;
-		this.matchLength = formatterMatch[0].length;
-		this.matchText = formatterMatch[0];
-		this.nextMatch = this.formatter.formatterRegExp.lastIndex;
-		// Figure out which formatter matched and call its handler
-		var t;
-		for(t=1; t<formatterMatch.length; t++) {
-			if(formatterMatch[t]) {
-				this.formatter.formatters[t-1].handler(this);
-				this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-				break;
-			}
-		}
-		// Get the next match
-		formatterMatch = this.formatter.formatterRegExp.exec(this.source);
-	}
-	// Output any text after the last match
-	if(this.nextMatch < this.source.length) {
-		this.outputText(this.output,this.nextMatch,this.source.length);
-		this.nextMatch = this.source.length;
-	}
-	// Restore the output pointer
-	this.output = oldOutput;
-};
-
-Wikifier.prototype.subWikifyTerm = function(output,terminatorRegExp)
-{
-	// subWikify can be indirectly recursive, so we need to save the old output pointer
-	var oldOutput = this.output;
-	this.output = output;
-	// Get the first matches for the formatter and terminator RegExps
-	terminatorRegExp.lastIndex = this.nextMatch;
-	var terminatorMatch = terminatorRegExp.exec(this.source);
-	this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-	var formatterMatch = this.formatter.formatterRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
-	while(terminatorMatch || formatterMatch) {
-		// Check for a terminator match before the next formatter match
-		if(terminatorMatch && (!formatterMatch || terminatorMatch.index <= formatterMatch.index)) {
-			// Output any text before the match
-			if(terminatorMatch.index > this.nextMatch)
-				this.outputText(this.output,this.nextMatch,terminatorMatch.index);
-			// Set the match parameters
-			this.matchText = terminatorMatch[1];
-			this.matchLength = terminatorMatch[1].length;
-			this.matchStart = terminatorMatch.index;
-			this.nextMatch = this.matchStart + this.matchLength;
-			// Restore the output pointer
-			this.output = oldOutput;
-			return;
-		}
-		// It must be a formatter match; output any text before the match
-		if(formatterMatch.index > this.nextMatch)
-			this.outputText(this.output,this.nextMatch,formatterMatch.index);
-		// Set the match parameters
-		this.matchStart = formatterMatch.index;
-		this.matchLength = formatterMatch[0].length;
-		this.matchText = formatterMatch[0];
-		this.nextMatch = this.formatter.formatterRegExp.lastIndex;
-		// Figure out which formatter matched and call its handler
-		var t;
-		for(t=1; t<formatterMatch.length; t++) {
-			if(formatterMatch[t]) {
-				this.formatter.formatters[t-1].handler(this);
-				this.formatter.formatterRegExp.lastIndex = this.nextMatch;
-				break;
-			}
-		}
-		// Get the next match
-		terminatorRegExp.lastIndex = this.nextMatch;
-		terminatorMatch = terminatorRegExp.exec(this.source);
-		formatterMatch = this.formatter.formatterRegExp.exec(terminatorMatch ? this.source.substr(0,terminatorMatch.index) : this.source);
-	}
-	// Output any text after the last match
-	if(this.nextMatch < this.source.length) {
-		this.outputText(this.output,this.nextMatch,this.source.length);
-		this.nextMatch = this.source.length;
-	}
-	// Restore the output pointer
-	this.output = oldOutput;
-};
-
-exports.Wikifier = Wikifier;
diff --git a/wikitest.js b/wikitest.js
index 139fdf83b..bbd51cab9 100644
--- a/wikitest.js
+++ b/wikitest.js
@@ -5,27 +5,23 @@ Wikifier test rig
 
 var Tiddler = require("./js/Tiddler.js").Tiddler,
 	TiddlyWiki = require("./js/TiddlyWiki.js").TiddlyWiki,
-	Formatter = require("./js/Formatter.js").Formatter,
-	Wikifier = require("./js/Wikifier.js").Wikifier,
 	utils = require("./js/Utils.js"),
 	util = require("util");
 
 var wikiTest = function(spec) {
 	var t,
 		store = new TiddlyWiki(),
-		formatter = new Formatter(),
-		wikifier = new Wikifier(store,formatter),
 		w;
 	for(t=0; t<spec.tiddlers.length; t++) {
 		store.addTiddler(new Tiddler(spec.tiddlers[t]));
 	}
 	for(t=0; t<spec.tests.length; t++) {
-		w = wikifier.wikify(store.getTiddlerText(spec.tests[t].tiddler));
+		w = store.getTiddler(spec.tests[t].tiddler).getParseTree().tree;
 		if(JSON.stringify(w) !== JSON.stringify(spec.tests[t].output)) {
 			console.error("Failed at tiddler: " + spec.tests[t].tiddler + " with JSON:\n" + util.inspect(w,false,8));
 		}
 	}
-}
+};
 
 wikiTest({ tiddlers: 
    [ { title: 'FirstTiddler',