Add more start-end range attributes for AST

core/modules/parsers/wikiparser/rules/extlink.js

@@ -31,6 +31,7 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
+    var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(this.match[0].substr(0,1) === "~") {
@@ -46,7 +47,7 @@ exports.parse = function() {
 				rel: {type: "string", value: "noopener noreferrer"}
 			},
 			children: [{
-				type: "text", text: this.match[0]
+				type: "text", text: this.match[0], start: start, end: this.parser.pos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/list.js

@@ -74,6 +74,7 @@ exports.parse = function() {
 		// Match the list marker
 		var reMatch = /([\*#;:>]+)/mg;
 		reMatch.lastIndex = this.parser.pos;
+		var start = this.parser.pos;
 		var match = reMatch.exec(this.parser.source);
 		if(!match || match.index !== this.parser.pos) {
 			break;
@@ -94,9 +95,21 @@ exports.parse = function() {
 			}
 			// Construct the list element or reuse the previous one at this level
 			if(listStack.length <= t) {
-				var listElement = {type: "element", tag: listInfo.listTag, children: [
+				var listElement = {
-					{type: "element", tag: listInfo.itemTag, children: []}
+					type: "element",
-				]};
+					tag: listInfo.listTag,
+					children: [
+						{
+							type: "element",
+							tag: listInfo.itemTag,
+							children: [],
+							start: start,
+							end: this.parser.pos,
+						}
+					],
+					start: start,
+					end: this.parser.pos,
+				};
 				// Link this list element into the last child item of the parent list item
 				if(t) {
 					var prevListItem = listStack[t-1].children[listStack[t-1].children.length-1];
@@ -105,7 +118,13 @@ exports.parse = function() {
 				// Save this element in the stack
 				listStack[t] = listElement;
 			} else if(t === (match[0].length - 1)) {
-				listStack[t].children.push({type: "element", tag: listInfo.itemTag, children: []});
+				listStack[t].children.push({
+					type: "element",
+					tag: listInfo.itemTag,
+					children: [],
+					start: start,
+					end: this.parser.pos,
+				});
 			}
 		}
 		if(listStack.length > match[0].length) {
@@ -118,6 +137,8 @@ exports.parse = function() {
 		this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
 		var tree = this.parser.parseInlineRun(/(\r?\n)/mg);
 		lastListItem.children.push.apply(lastListItem.children,tree);
+		lastListItem.end = this.parser.pos;
+		listStack[listStack.length-1].end = this.parser.pos;
 		if(classes.length > 0) {
 			$tw.utils.addClassToParseTreeNode(lastListItem,classes.join(" "));
 		}

core/modules/parsers/wikiparser/rules/prettyextlink.js

@@ -97,12 +97,15 @@ exports.parseLink = function(source,pos) {
 	}
 	// Pull out the tooltip and URL
 	var tooltip, URL;
+	textNode.start = pos;
 	if(splitPos) {
 		URL = source.substring(splitPos + 1,closePos).trim();
 		textNode.text = source.substring(pos,splitPos).trim();
+		textNode.end = splitPos;
 	} else {
 		URL = source.substring(pos,closePos).trim();
 		textNode.text = URL;
+		textNode.end = closePos;
 	}
 	node.attributes.href = {type: "string", value: URL};
 	node.attributes.target = {type: "string", value: "_blank"};

core/modules/parsers/wikiparser/rules/prettylink.js

@@ -29,10 +29,15 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Process the link
 	var text = this.match[1],
-		link = this.match[2] || text;
+		link = this.match[2] || text,
+		textEndPos = this.parser.source.indexOf("|", start);
+	if (textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
+		textEndPos = this.matchRegExp.lastIndex - 2;
+	}
 	if($tw.utils.isLinkExternal(link)) {
 		return [{
 			type: "element",
@@ -44,7 +49,7 @@ exports.parse = function() {
 				rel: {type: "string", value: "noopener noreferrer"}
 			},
 			children: [{
-				type: "text", text: text
+				type: "text", text: text, start: start, end: textEndPos
 			}]
 		}];
 	} else {
@@ -54,7 +59,7 @@ exports.parse = function() {
 				to: {type: "string", value: link}
 			},
 			children: [{
-				type: "text", text: text
+				type: "text", text: text, start: start, end: textEndPos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/quoteblock.js

@@ -9,7 +9,7 @@ Wiki text rule for quote blocks. For example:
 	<<<.optionalClass(es) optional cited from
 	a quote
 	<<<
-	
+
 	<<<.optionalClass(es)
 	a quote
 	<<< optional cited from
@@ -20,11 +20,11 @@ Quotes can be quoted by putting more <s
 ```
 	<<<
 	Quote Level 1
-	
+
 	<<<<
 	QuoteLevel 2
 	<<<<
-	
+
 	<<<
 ```
 
@@ -50,30 +50,38 @@ exports.parse = function() {
 	var reEndString = "^" + this.match[1] + "(?!<)";
 	// Move past the <s
 	this.parser.pos = this.matchRegExp.lastIndex;
-	
+
 	// Parse any classes, whitespace and then the optional cite itself
 	classes.push.apply(classes, this.parser.parseClasses());
 	this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
+	var citeStart = this.parser.pos;
 	var cite = this.parser.parseInlineRun(/(\r?\n)/mg);
+	var citeEnd = this.parser.pos;
 	// before handling the cite, parse the body of the quote
-	var tree= this.parser.parseBlocks(reEndString);
+	var tree = this.parser.parseBlocks(reEndString);
 	// If we got a cite, put it before the text
 	if(cite.length > 0) {
 		tree.unshift({
 			type: "element",
 			tag: "cite",
-			children: cite
+			children: cite,
+			start: citeStart,
+			end: citeEnd
 		});
 	}
 	// Parse any optional cite
 	this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
+	citeStart = this.parser.pos;
 	cite = this.parser.parseInlineRun(/(\r?\n)/mg);
+	citeEnd = this.parser.pos;
 	// If we got a cite, push it
 	if(cite.length > 0) {
 		tree.push({
 			type: "element",
 			tag: "cite",
-			children: cite
+			children: cite,
+			start: citeStart,
+			end: citeEnd
 		});
 	}
 	// Return the blockquote element

core/modules/parsers/wikiparser/rules/syslink.js

@@ -29,10 +29,11 @@ exports.init = function(parser) {
 exports.parse = function() {
 	var match = this.match[0];
 	// Move past the match
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(match.substr(0,1) === "~") {
-		return [{type: "text", text: match.substr(1)}];
+		return [{type: "text", text: match.substr(1), start: start+1, end: this.parser.pos}];
 	} else {
 		return [{
 			type: "link",
@@ -41,10 +42,12 @@ exports.parse = function() {
 			},
 			children: [{
 				type: "text",
-				text: match
+				text: match,
+				start: start,
+				end: this.parser.pos
 			}]
 		}];
 	}
 };
 
-})();
+})();

core/modules/parsers/wikiparser/rules/table.js

@@ -150,7 +150,7 @@ exports.parse = function() {
 		} else {
 			// Otherwise, create a new row if this one is of a different type
 			if(rowType !== currRowType) {
-				rowContainer = {type: "element", tag: rowContainerTypes[rowType], children: []};
+				rowContainer = {type: "element", tag: rowContainerTypes[rowType], children: [], start: this.parser.pos, end: this.parser.pos};
 				table.children.push(rowContainer);
 				currRowType = rowType;
 			}
@@ -178,6 +178,7 @@ exports.parse = function() {
 				// Increment the row count
 				rowCount++;
 			}
+			rowContainer.end = this.parser.pos;
 		}
 		rowMatch = rowRegExp.exec(this.parser.source);
 	}

core/modules/parsers/wikiparser/rules/typedblock.js

@@ -46,6 +46,7 @@ exports.parse = function() {
 		renderType = this.match[2];
 	// Move past the match
 	this.parser.pos = this.matchRegExp.lastIndex;
+	var start = this.parser.pos;
 	// Look for the end of the block
 	reEnd.lastIndex = this.parser.pos;
 	var match = reEnd.exec(this.parser.source),
@@ -74,7 +75,9 @@ exports.parse = function() {
 			tag: "pre",
 			children: [{
 				type: "text",
-				text: text
+				text: text,
+				start: start,
+				end: this.parser.pos
 			}]
 		}];
 	}

core/modules/parsers/wikiparser/rules/wikilink.js

@@ -36,6 +36,7 @@ exports.parse = function() {
 	// Get the details of the match
 	var linkText = this.match[0];
 	// Move past the macro call
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// If the link starts with the unwikilink character then just output it as plain text
 	if(linkText.substr(0,1) === $tw.config.textPrimitives.unWikiLink) {
@@ -57,7 +58,9 @@ exports.parse = function() {
 		},
 		children: [{
 			type: "text",
-			text: linkText
+			text: linkText,
+			start: start,
+			end: this.parser.pos
 		}]
 	}];
 };

core/modules/parsers/wikiparser/wikiparser.js

@@ -254,7 +254,7 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 	var start = this.pos;
 	var children = this.parseInlineRun(terminatorRegExp);
 	var end = this.pos;
-	return [{type: "element", tag: "p", children: children, start: start, end: end, parseRule: null }];
+	return [{type: "element", tag: "p", children: children, start: start, end: end, parseRule: undefined }];
 };
 
 /*
@@ -354,7 +354,7 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
 			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 		}
 		var rule = Object.getPrototypeOf(nextMatch.rule);
-		for (const node of subTree) node.rule = rule;
+        $tw.utils.each(subTree, function (node) { node.rule = rule; });
 		tree.push.apply(tree,subTree);
 		// Look for the next run rule
 		nextMatch = this.findNextMatch(this.inlineRules,this.pos);
@@ -441,7 +441,7 @@ WikiParser.prototype.pushTextWidget = function(array,text,start,end) {
 		text = $tw.utils.trim(text);
 	}
 	if(text) {
-		array.push({type: "text", text: text, start: start, end: end, parseRule: null});
+		array.push({type: "text", text: text, start: start, end: end, parseRule: undefined});
 	}
 };