From 18f8b7266e841f2ebec7e4b656b3ce6a915fadc2 Mon Sep 17 00:00:00 2001
From: Jeremy Ruston <jeremy@osmosoft.com>
Date: Sat, 15 Jun 2013 15:12:05 +0100
Subject: [PATCH] Refactor the HTML element parser

The purpose is to allow attributes to be specified as macro
invocations. For example `<div myattr=<<mymacro param1 param3>>>`. The
parser needed sprucing up in order to copy with the nesting of angle
brackets. The refactoring has been done with an eye on using the same
technique in the filter expression parser (which is pretty messy at the
moment -- it throws exceptions for syntax errors, which is bad). Later
I'm hoping to extend the technique to create a more flexible table
parser.
---
 core/modules/parsers/wikiparser/rules/html.js | 394 +++++++++++++++---
 .../test/tiddlers/tests/test-html-parser.js   | 224 +++++++++-
 2 files changed, 558 insertions(+), 60 deletions(-)

diff --git a/core/modules/parsers/wikiparser/rules/html.js b/core/modules/parsers/wikiparser/rules/html.js
index 226c762bf..52acbe663 100644
--- a/core/modules/parsers/wikiparser/rules/html.js
+++ b/core/modules/parsers/wikiparser/rules/html.js
@@ -28,72 +28,366 @@ exports.types = {inline: true, block: true};
 
 exports.init = function(parser) {
 	this.parser = parser;
-	// Regexp to match
-	if(this.is.block) {
-		this.matchRegExp = /<([A-Za-z\$]+)(\s*[^>]*?)(\/)?>(\r?\n)/mg;
-	} else {
-		this.matchRegExp = /<([A-Za-z\$]+)(\s*[^>]*?)(?:(\/>)|(?:>(\r?\n)?))/mg;
-	}
+};
+
+exports.findNextMatch = function(startPos) {
+	// Find the next tag
+	this.nextTag = this.findNextTag(this.parser.source,startPos,{
+		requireLineBreak: this.is.block
+	});
+	return this.nextTag ? this.nextTag.start : undefined;
 };
 
 /*
 Parse the most recent match
 */
 exports.parse = function() {
-	// Get all the details of the match in case this parser is called recursively
-	var tagName = this.match[1],
-		attributeString = this.match[2],
-		isSelfClosing = !!this.match[3],
-		hasLineBreak = !!this.match[4];
-	// Move past the tag name and parameters
-	this.parser.pos = this.matchRegExp.lastIndex;
-	var reAttr = /\s*([A-Za-z\-_]+)(?:\s*=\s*(?:("[^"]*")|('[^']*')|(\{\{[^\}]*\}\})|([^"'\s]+)))?/mg;
-	// Process the attributes
-	var attrMatch = reAttr.exec(attributeString),
-		attributes = {};
-	while(attrMatch) {
-		var name = attrMatch[1],
-			value;
-		if(attrMatch[2]) { // Double quoted
-			value = {type: "string", value: attrMatch[2].substring(1,attrMatch[2].length-1)};
-		} else if(attrMatch[3]) { // Single quoted
-			value = {type: "string", value: attrMatch[3].substring(1,attrMatch[3].length-1)};
-		} else if(attrMatch[4]) { // Double curly brace quoted
-			value = {type: "indirect", textReference: attrMatch[4].substr(2,attrMatch[4].length-4)};
-		} else if(attrMatch[5]) { // Unquoted
-			value = {type: "string", value: attrMatch[5]};
-		} else { // Valueless
-			value = {type: "string", value: "true"}; // TODO: We should have a way of indicating we want an attribute without a value
-		}
-		attributes[name] = value;
-		attrMatch = reAttr.exec(attributeString);
-	}
-	// Process the end tag
-	if(!isSelfClosing && $tw.config.htmlVoidElements.indexOf(tagName) === -1) {
-		var reEndString = "</" + $tw.utils.escapeRegExp(tagName) + ">",
-			reEnd = new RegExp("(" + reEndString + ")","mg"),
-			content;
+	// Retrieve the most recent match so that recursive calls don't overwrite it
+	var tag = this.nextTag;
+	this.nextTag = null;
+	// Advance the parser position to past the tag
+	this.parser.pos = tag.end;
+	// Check for a following linebreak
+	var hasLineBreak = !tag.isSelfClosing && !!this.parseTokenRegExp(this.parser.source,this.parser.pos,/(\r?\n)/g);
+	// Set whether we're in block mode
+	tag.isBlock = this.is.block || hasLineBreak;
+	// Parse the body if we need to
+	if(!tag.isSelfClosing && $tw.config.htmlVoidElements.indexOf(tag.tag) === -1) {
+			var reEndString = "</" + $tw.utils.escapeRegExp(tag.tag) + ">",
+				reEnd = new RegExp("(" + reEndString + ")","mg");
 		if(hasLineBreak) {
-			content = this.parser.parseBlocks(reEndString);
+			tag.children = this.parser.parseBlocks(reEndString);
 		} else {
-			content = this.parser.parseInlineRun(reEnd);
+			tag.children = this.parser.parseInlineRun(reEnd);
 		}
 		reEnd.lastIndex = this.parser.pos;
 		var endMatch = reEnd.exec(this.parser.source);
 		if(endMatch && endMatch.index === this.parser.pos) {
 			this.parser.pos = endMatch.index + endMatch[0].length;
 		}
-	} else {
-		content = [];
 	}
-	var element = {
-		type: "element",
-		tag: tagName,
-		isBlock: this.is.block || hasLineBreak,
-		attributes: attributes,
-		children: content
+	// Return the tag
+	return [tag];
+};
+
+/*
+Look for a whitespace token. Returns null if not found, otherwise returns {type: "whitespace", start:, end:,}
+*/
+exports.parseWhiteSpace = function(source,pos) {
+	var node = {
+		type: "whitespace",
+		start: pos
 	};
-	return [element];
+	var re = /(\s)+/g;
+	re.lastIndex = pos;
+	var match = re.exec(source);
+	if(match && match.index === pos) {
+		node.end = pos + match[0].length;
+		return node;
+	}
+	return null;
+};
+
+/*
+Convenience wrapper for parseWhiteSpace
+*/
+exports.skipWhiteSpace = function(source,pos) {
+	var whitespace = this.parseWhiteSpace(source,pos);
+	if(whitespace) {
+		return whitespace.end;
+	}
+	return pos;
+};
+
+/*
+Look for a given string token. Returns null if not found, otherwise returns {type: "token", value:, start:, end:,}
+*/
+exports.parseTokenString = function(source,pos,token) {
+	var match = source.indexOf(token,pos) === pos;
+	if(match) {
+		return {
+			type: "token",
+			value: token,
+			start: pos,
+			end: pos + token.length
+		};
+	}
+	return null;
+};
+
+/*
+Look for a token matching a regex. Returns null if not found, otherwise returns {type: "regexp", match:, start:, end:,}
+*/
+exports.parseTokenRegExp = function(source,pos,reToken) {
+	var node = {
+		type: "regexp",
+		start: pos
+	};
+	reToken.lastIndex = pos;
+	node.match = reToken.exec(source);
+	if(node.match && node.match.index === pos) {
+		node.end = pos + node.match[0].length;
+		return node;
+	} else {
+		return null;
+	}
+};
+
+/*
+Look for a string literal. Returns null if not found, otherwise returns {type: "string", value:, start:, end:,}
+*/
+exports.parseStringLiteral = function(source,pos) {
+	var node = {
+		type: "string",
+		start: pos
+	};
+	var reString = /(?:"([^"]*)")|(?:'([^']*)')/g;
+	reString.lastIndex = pos;
+	var match = reString.exec(source);
+	if(match && match.index === pos) {
+		node.value = match[1] === undefined ? match[2] : match[1];
+		node.end = pos + match[0].length;
+		return node;
+	} else {
+		return null;
+	}
+};
+
+/*
+Look for a macro invocation parameter. Returns null if not found, or {type: "macro-parameter", name:, value:, start:, end:}
+*/
+exports.parseMacroParameter = function(source,pos) {
+	var node = {
+		type: "macro-parameter",
+		start: pos
+	}
+	// Define our regexp
+	var reMacroParameter = /(?:([A-Za-z0-9\-_]+)\s*:)?(?:\s*(?:"([^"]*)"|'([^']*)'|\[\[([^\]]*)\]\]|([^\s>"'=]+)))/g;
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for the parameter
+	var token = this.parseTokenRegExp(source,pos,reMacroParameter);
+	if(!token) {
+		return null;
+	}
+	pos = token.end;
+	// Get the parameter details
+	node.value = token.match[2] !== undefined ? token.match[2] : (
+					token.match[3] !== undefined ? token.match[3] : (
+						token.match[4] !== undefined ? token.match[4] : (
+							token.match[5] !== undefined ? token.match[5] : (
+								""
+							)
+						)
+					)
+				);
+	if(token.match[1]) {
+		node.name = token.match[1];
+	}
+	// Update the end position
+	node.end = pos;
+	return node;
+};
+
+/*
+Look for a macro invocation. Returns null if not found, or {type: "macro-invocation", name:, parameters:, start:, end:}
+*/
+exports.parseMacroInvocation = function(source,pos) {
+	var node = {
+		type: "macro-invocation",
+		start: pos,
+		parameters: []
+	}
+	// Define our regexps
+	var reMacroName = /([^\s>"'=]+)/g;
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for a double less than sign
+	var token = this.parseTokenString(source,pos,"<<");
+	if(!token) {
+		return null;
+	}
+	pos = token.end;
+	// Get the macro name
+	var name = this.parseTokenRegExp(source,pos,reMacroName);
+	if(!name) {
+		return null;
+	}
+	node.name = name.match[1];
+	pos = name.end;
+	// Process parameters
+	var parameter = this.parseMacroParameter(source,pos);
+	while(parameter) {
+		node.parameters.push(parameter);
+		pos = parameter.end;
+		// Get the next parameter
+		parameter = this.parseMacroParameter(source,pos);
+	}
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for a double greater than sign
+	token = this.parseTokenString(source,pos,">>");
+	if(!token) {
+		return null;
+	}
+	pos = token.end;
+	// Update the end position
+	node.end = pos;
+	return node;
+};
+
+/*
+Look for an HTML attribute definition. Returns null if not found, otherwise returns {type: "attribute", name:, valueType: "string|indirect|macro", value:, start:, end:,}
+*/
+exports.parseAttribute = function(source,pos) {
+	var node = {
+		start: pos
+	};
+	// Define our regexps
+	var reAttributeName = /([^\/\s>"'=]+)/g,
+		reUnquotedAttribute = /([^\/\s<>"'=]+)/g,
+		reIndirectValue = /\{\{([^\}]+)\}\}/g;
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Get the attribute name
+	var name = this.parseTokenRegExp(source,pos,reAttributeName);
+	if(!name) {
+		return null;
+	}
+	node.name = name.match[1];
+	pos = name.end;
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for an equals sign
+	var token = this.parseTokenString(source,pos,"=");
+	if(token) {
+		pos = token.end;
+		// Skip whitespace
+		pos = this.skipWhiteSpace(source,pos);
+		// Look for a string literal
+		var stringLiteral = this.parseStringLiteral(source,pos);
+		if(stringLiteral) {
+			pos = stringLiteral.end;
+			node.type = "string";
+			node.value = stringLiteral.value;
+		} else {
+			// Look for an indirect value
+			var indirectValue = this.parseTokenRegExp(source,pos,reIndirectValue);
+			if(indirectValue) {
+				pos = indirectValue.end;
+				node.type = "indirect";
+				node.textReference = indirectValue.match[1];
+			} else {
+				// Look for a unquoted value
+				var unquotedValue = this.parseTokenRegExp(source,pos,reUnquotedAttribute);
+				if(unquotedValue) {
+					pos = unquotedValue.end;
+					node.type = "string";
+					node.value = unquotedValue.match[1];
+				} else {
+					// Look for a macro invocation value
+					var macroInvocation = this.parseMacroInvocation(source,pos);
+					if(macroInvocation) {
+						pos = macroInvocation.end;
+						node.type = "macro";
+						node.value = macroInvocation;
+					} else {
+						node.type = "string";
+						node.value = "true";
+					}
+				}
+			}
+		}
+	} else {
+		node.type = "string";
+		node.value = "true";
+	}
+	// Update the end position
+	node.end = pos;
+	return node;
+};
+
+/*
+Look for an HTML tag. Returns null if not found, otherwise returns {type: "tag", name:, attributes: [], isSelfClosing:, start:, end:,}
+*/
+exports.parseTag = function(source,pos,options) {
+	options = options || {};
+	var token,
+		node = {
+			type: "element",
+			start: pos,
+			attributes: {}
+		};
+	// Define our regexps
+	var reTagName = /([a-zA-Z\-\$]+)/g;
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for a less than sign
+	token = this.parseTokenString(source,pos,"<");
+	if(!token) {
+		return null;
+	}
+	pos = token.end;
+	// Get the tag name
+	token = this.parseTokenRegExp(source,pos,reTagName);
+	if(!token) {
+		return null;
+	}
+	node.tag = token.match[1];
+	pos = token.end;
+	// Process attributes
+	var attribute = this.parseAttribute(source,pos);
+	while(attribute) {
+		node.attributes[attribute.name] = attribute;
+		pos = attribute.end;
+		// Get the next attribute
+		attribute = this.parseAttribute(source,pos);
+	}
+	// Skip whitespace
+	pos = this.skipWhiteSpace(source,pos);
+	// Look for a closing slash
+	token = this.parseTokenString(source,pos,"/");
+	if(token) {
+		pos = token.end;
+		node.isSelfClosing = true;
+	}
+	// Look for a greater than sign
+	token = this.parseTokenString(source,pos,">");
+	if(!token) {
+		return null;
+	}
+	pos = token.end;
+	// Check for a required line break
+	if(options.requireLineBreak) {
+		token = this.parseTokenRegExp(source,pos,/(\r?\n)/g);
+		if(!token) {
+			return null;
+		}
+	}
+	// Update the end position
+	node.end = pos;
+	return node;
+};
+
+exports.findNextTag = function(source,pos,options) {
+	// A regexp for finding candidate HTML tags
+	var reLookahead = /<([a-zA-Z\-\$]+)/g;
+	// Find the next candidate
+	reLookahead.lastIndex = pos;
+	var match = reLookahead.exec(source);
+	while(match) {
+		// Try to parse the candidate as a tag
+		var tag = this.parseTag(source,match.index,options);
+		// Return success
+		if(tag) {
+			return tag;
+		}
+		// Look for the next match
+		reLookahead.lastIndex = match.index + 1;
+		match = reLookahead.exec(source);
+	}
+	// Failed
+	return null;
 };
 
 })();
diff --git a/editions/test/tiddlers/tests/test-html-parser.js b/editions/test/tiddlers/tests/test-html-parser.js
index f8db75164..158079ab1 100644
--- a/editions/test/tiddlers/tests/test-html-parser.js
+++ b/editions/test/tiddlers/tests/test-html-parser.js
@@ -3,7 +3,7 @@ title: test-html-parser.js
 type: application/javascript
 tags: [[$:/tags/test-spec]]
 
-Tests the parse rule for HTML elements
+Tests for the internal components of the HTML tag parser
 
 \*/
 (function(){
@@ -12,6 +12,195 @@ Tests the parse rule for HTML elements
 /*global $tw: false */
 "use strict";
 
+function FakeParser() {
+
+}
+
+$tw.utils.extend(FakeParser.prototype,require("$:/core/modules/parsers/wikiparser/rules/html.js"));
+
+describe("HTML tag new parser tests", function() {
+
+	var parser = new FakeParser();
+
+	it("should parse whitespace", function() {
+		expect(parser.parseWhiteSpace("p  ",0)).toEqual(
+			null
+		);
+		expect(parser.parseWhiteSpace("p  ",1)).toEqual(
+			{ type : 'whitespace', start : 1, end : 3 }
+		);
+	});
+
+	it("should parse string tokens", function() {
+		expect(parser.parseTokenString("p=  ",0,"=")).toEqual(
+			null
+		);
+		expect(parser.parseTokenString("p=  ",1,"=")).toEqual(
+			{ type : 'token', value : '=', start : 1, end : 2 }
+		);
+	});
+
+	it("should parse regexp tokens", function() {
+		expect(parser.parseTokenRegExp("p='  ",0,/(=(?:'|"))/)).toEqual(
+			null
+		);
+		expect(parser.parseTokenRegExp("p='  ",1,/(=(?:'|"))/g).match[0]).toEqual(
+			'=\''
+		);
+		expect(parser.parseTokenRegExp("p=blah  ",2,/([^\s>]+)/g).match[0]).toEqual(
+			'blah'
+		);
+	});
+
+	it("should parse string literals", function() {
+		expect(parser.parseStringLiteral("p='blah'  ",0)).toEqual(
+			null
+		);
+		expect(parser.parseStringLiteral("p='blah'  ",2)).toEqual(
+			{ type : 'string', start : 2, value : 'blah', end : 8 }
+		);
+		expect(parser.parseStringLiteral("p=''  ",2)).toEqual(
+			{ type : 'string', start : 2, value : '', end : 4 }
+		);
+		expect(parser.parseStringLiteral("p=\"blah'  ",2)).toEqual(
+			null
+		);
+		expect(parser.parseStringLiteral("p=\"\"  ",2)).toEqual(
+			{ type : 'string', start : 2, value : '', end : 4 }
+		);
+	});
+
+	it("should parse macro parameters", function() {
+		expect(parser.parseMacroParameter("me",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'me', end : 2 }
+		);
+		expect(parser.parseMacroParameter("me:one",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'one', name : 'me', end : 6 }
+		);
+		expect(parser.parseMacroParameter("me:'one two three'",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'one two three', name : 'me', end : 18 }
+		);
+		expect(parser.parseMacroParameter("'one two three'",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'one two three', end : 15 }
+		);
+		expect(parser.parseMacroParameter("me:[[one two three]]",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'one two three', name : 'me', end : 20 }
+		);
+		expect(parser.parseMacroParameter("[[one two three]]",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'one two three', end : 17 }
+		);
+		expect(parser.parseMacroParameter("myparam>",0)).toEqual(
+			{ type : 'macro-parameter', start : 0, value : 'myparam', end : 7 }
+		);
+	});
+
+	it("should parse macro invocations", function() {
+		expect(parser.parseMacroInvocation("<<mymacro",0)).toEqual(
+			null
+		);
+		expect(parser.parseMacroInvocation("<<mymacro>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [  ], name : 'mymacro', end : 11 }
+		);
+		expect(parser.parseMacroInvocation("<<mymacro one two three>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [ { type : 'macro-parameter', start : 9, value : 'one', end : 13 }, { type : 'macro-parameter', start : 13, value : 'two', end : 17 }, { type : 'macro-parameter', start : 17, value : 'three', end : 23 } ], name : 'mymacro', end : 25 }
+		);
+		expect(parser.parseMacroInvocation("<<mymacro p:one q:two three>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [ { type : 'macro-parameter', start : 9, value : 'one', name : 'p', end : 15 }, { type : 'macro-parameter', start : 15, value : 'two', name : 'q', end : 21 }, { type : 'macro-parameter', start : 21, value : 'three', end : 27 } ], name : 'mymacro', end : 29 }
+		);
+		expect(parser.parseMacroInvocation("<<mymacro 'one two three'>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [ { type : 'macro-parameter', start : 9, value : 'one two three', end : 25 } ], name : 'mymacro', end : 27 } 
+		);
+		expect(parser.parseMacroInvocation("<<mymacro r:'one two three'>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [ { type : 'macro-parameter', start : 9, value : 'one two three', name : 'r', end : 27 } ], name : 'mymacro', end : 29 } 
+		);
+		expect(parser.parseMacroInvocation("<<myMacro one:two three:'four and five'>>",0)).toEqual(
+			{ type : 'macro-invocation', start : 0, parameters : [ { type : 'macro-parameter', start : 9, value : 'two', name : 'one', end : 17 }, { type : 'macro-parameter', start : 17, value : 'four and five', name : 'three', end : 39 } ], name : 'myMacro', end : 41 } 
+		);
+	});
+
+	it("should parse HTML attributes", function() {
+		expect(parser.parseAttribute("p='blah'  ",1)).toEqual(
+			null
+		);
+		expect(parser.parseAttribute("p='blah'  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 8 }
+		);
+		expect(parser.parseAttribute("p=\"blah\"  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 8 }
+		);
+		expect(parser.parseAttribute("p=blah  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 6 }
+		);
+		expect(parser.parseAttribute("p =blah  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 7 }
+		);
+		expect(parser.parseAttribute("p= blah  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 7 }
+		);
+		expect(parser.parseAttribute("p = blah  ",0)).toEqual(
+			{ type : 'string', start : 0, name : 'p', value : 'blah', end : 8 }
+		);
+		expect(parser.parseAttribute("p = >blah  ",0)).toEqual(
+			{ type : 'string', value : 'true', start : 0, name : 'p', end : 4 }
+		);
+		expect(parser.parseAttribute(" attrib1>",0)).toEqual(
+			{ type : 'string', value : 'true', start : 0, name : 'attrib1', end : 8 }
+		);
+	});
+
+	it("should parse HTML tags", function() {
+		expect(parser.parseTag("<mytag>",1)).toEqual(
+			null
+		);
+		expect(parser.parseTag("</mytag>",0)).toEqual(
+			null
+		);
+		expect(parser.parseTag("<mytag>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : [  ], tag : 'mytag', end : 7 }
+		);
+		expect(parser.parseTag("<mytag attrib1>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib1 : { type : 'string', value : 'true', start : 6, name : 'attrib1', end : 14 } }, tag : 'mytag', end : 15 }
+		);
+		expect(parser.parseTag("<mytag attrib1/>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib1 : { type : 'string', value : 'true', start : 6, name : 'attrib1', end : 14 } }, tag : 'mytag', isSelfClosing : true, end : 16 }
+		);
+		expect(parser.parseTag("<$view field=\"title\" format=\"link\"/>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { field : { start : 6, name : 'field', type : 'string', value : 'title', end : 20 }, format : { start : 20, name : 'format', type : 'string', value : 'link', end : 34 } }, tag : '$view', isSelfClosing : true, end : 36 }
+		);
+		expect(parser.parseTag("<mytag attrib1='something'>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib1 : { type : 'string', start : 6, name : 'attrib1', value : 'something', end : 26 } }, tag : 'mytag', end : 27 }
+		);
+		expect(parser.parseTag("<$mytag attrib1='something' attrib2=else thing>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib1 : { type : 'string', start : 7, name : 'attrib1', value : 'something', end : 27 }, attrib2 : { type : 'string', start : 27, name : 'attrib2', value : 'else', end : 40 }, thing : { type : 'string', start : 40, name : 'thing', value : 'true', end : 46 } }, tag : '$mytag', end : 47 }
+		);
+		expect(parser.parseTag("< $mytag attrib1='something' attrib2=else thing>",0)).toEqual(
+			null
+		);
+		expect(parser.parseTag("<$mytag attrib3=<<myMacro one:two three:'four and five'>>>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib3 : { type : 'macro', start : 7, name : 'attrib3', value : { type : 'macro-invocation', start : 16, parameters : [ { type : 'macro-parameter', start : 25, value : 'two', name : 'one', end : 33 }, { type : 'macro-parameter', start : 33, value : 'four and five', name : 'three', end : 55 } ], name : 'myMacro', end : 57 }, end : 57 } }, tag : '$mytag', end : 58 }
+		);
+		expect(parser.parseTag("<$mytag attrib1='something' attrib2=else thing attrib3=<<myMacro one:two three:'four and five'>>>",0)).toEqual(
+			{ type : 'element', start : 0, attributes : { attrib1 : { type : 'string', start : 7, name : 'attrib1', value : 'something', end : 27 }, attrib2 : { type : 'string', start : 27, name : 'attrib2', value : 'else', end : 40 }, thing : { type : 'string', start : 40, name : 'thing', value : 'true', end : 47 }, attrib3 : { type : 'macro', start : 47, name : 'attrib3', value : { type : 'macro-invocation', start : 55, parameters : [ { type : 'macro-parameter', start : 64, value : 'two', name : 'one', end : 72 }, { type : 'macro-parameter', start : 72, value : 'four and five', name : 'three', end : 94 } ], name : 'myMacro', end : 96 }, end : 96 } }, tag : '$mytag', end : 97 }
+		);
+	});
+
+	it("should find and parse HTML tags", function() {
+		expect(parser.findNextTag("<something <mytag>",1)).toEqual(
+			{ type : 'element', start : 11, attributes : {  }, tag : 'mytag', end : 18 }
+		);
+		expect(parser.findNextTag("something else </mytag>",0)).toEqual(
+			null
+		);
+		expect(parser.findNextTag("<<some other stuff>> <mytag>",0)).toEqual(
+			{ type : 'element', start : 1, attributes : { other : { type : 'string', value : 'true', start : 6, name : 'other', end : 13 }, stuff : { type : 'string', value : 'true', start : 13, name : 'stuff', end : 18 } }, tag : 'some', end : 19 }
+		);
+		expect(parser.findNextTag("<<some other stuff>> <mytag>",2)).toEqual(
+			{ type : 'element', start : 21, attributes : {  }, tag : 'mytag', end : 28 }
+		);
+	});
+
+});
+
 describe("HTML tag parser tests", function() {
 
 	// Create a wiki
@@ -25,7 +214,7 @@ describe("HTML tag parser tests", function() {
 	it("should parse unclosed tags", function() {
 		expect(parse("<br>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'br', isBlock : false, attributes : {  }, children : [  ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'br', isBlock : false, attributes : {  }, start : 0, end : 4 } ] } ]
 
 		);
 		expect(parse("</br>")).toEqual(
@@ -35,42 +224,57 @@ describe("HTML tag parser tests", function() {
 		);
 		expect(parse("<div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [  ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [  ], start : 0, end : 5 } ] } ]
 
 		);
 		expect(parse("<div/>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [  ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isSelfClosing : true, isBlock : false, attributes : {  }, start : 0, end : 6 } ] } ]
 
 		);
 		expect(parse("<div></div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [  ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [ ], start : 0, end : 5 } ] } ]
 
 		);
 		expect(parse("<div>some text</div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [ { type : 'text', text : 'some text' } ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : {  }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 5 } ] } ]
 
 		);
 		expect(parse("<div attribute>some text</div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', value : 'true' } }, children : [ { type : 'text', text : 'some text' } ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', value : 'true', start : 4, end : 14, name: 'attribute' } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 15 } ] } ]
 
 		);
 		expect(parse("<div attribute='value'>some text</div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', value : 'value' } }, children : [ { type : 'text', text : 'some text' } ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', name: 'attribute', value : 'value', start: 4, end: 22 } }, children : [ { type : 'text', text : 'some text' } ], start: 0, end: 23 } ] } ]
 
 		);
 		expect(parse("<div attribute={{TiddlerTitle}}>some text</div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', textReference : 'TiddlerTitle' } }, children : [ { type : 'text', text : 'some text' } ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name: 'attribute', textReference : 'TiddlerTitle', start : 4, end : 31 } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 32 } ] } ]
+
+		);
+		expect(parse("<$reveal state='$:/temp/search' type='nomatch' text=''>")).toEqual(
+
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : { state : { start : 8, name : 'state', type : 'string', value : '$:/temp/search', end : 31 }, type : { start : 31, name : 'type', type : 'string', value : 'nomatch', end : 46 }, text : { start : 46, name : 'text', type : 'string', value : '', end : 54 } }, tag : '$reveal', end : 55, children : [  ], isBlock : false } ] } ]
 
 		);
 		expect(parse("<div attribute={{TiddlerTitle!!field}}>some text</div>")).toEqual(
 
-			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', textReference : 'TiddlerTitle!!field' } }, children : [ { type : 'text', text : 'some text' } ] } ] } ]
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name : 'attribute', textReference : 'TiddlerTitle!!field', start : 4, end : 38 } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 39 } ] } ]
+
+		);
+		expect(parse("<div attribute={{TiddlerTitle!!field}}>\nsome text</div>")).toEqual(
+
+			[ { type : 'element', start : 0, attributes : { attribute : { start : 4, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 38 } }, tag : 'div', end : 39, isBlock : true, children : [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'some text' } ] } ] } ]
+
+		);
+		expect(parse("<div><div attribute={{TiddlerTitle!!field}}>\nsome text</div></div>")).toEqual(
+
+			[ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : {  }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'some text' } ] } ] } ] } ] } ]
 
 		);
 	});