diff --git a/core/modules/parsers/parseutils.js b/core/modules/parsers/parseutils.js
index 1f86dd909..2a2e36309 100644
--- a/core/modules/parsers/parseutils.js
+++ b/core/modules/parsers/parseutils.js
@@ -114,7 +114,7 @@ exports.parseStringLiteral = function(source,pos) {
var match = reString.exec(source);
if(match && match.index === pos) {
node.value = match[1] !== undefined ? match[1] :(
- match[2] !== undefined ? match[2] : match[3]
+ match[2] !== undefined ? match[2] : match[3]
);
node.end = pos + match[0].length;
return node;
diff --git a/core/modules/parsers/wikiparser/rules/codeblock.js b/core/modules/parsers/wikiparser/rules/codeblock.js
index 262038f87..6c3480566 100644
--- a/core/modules/parsers/wikiparser/rules/codeblock.js
+++ b/core/modules/parsers/wikiparser/rules/codeblock.js
@@ -29,13 +29,16 @@ exports.init = function(parser) {
exports.parse = function() {
var reEnd = /(\r?\n```$)/mg;
+ var languageStart = this.parser.pos + 3,
+ languageEnd = languageStart + this.match[1].length;
// Move past the match
this.parser.pos = this.matchRegExp.lastIndex;
// Look for the end of the block
reEnd.lastIndex = this.parser.pos;
var match = reEnd.exec(this.parser.source),
- text;
+ text,
+ codeStart = this.parser.pos;
// Process the block
if(match) {
text = this.parser.source.substring(this.parser.pos,match.index);
@@ -48,8 +51,8 @@ exports.parse = function() {
return [{
type: "codeblock",
attributes: {
- code: {type: "string", value: text},
- language: {type: "string", value: this.match[1]}
+ code: {type: "string", value: text, start: codeStart, end: this.parser.pos},
+ language: {type: "string", value: this.match[1], start: languageStart, end: languageEnd}
}
}];
};
diff --git a/core/modules/parsers/wikiparser/rules/codeinline.js b/core/modules/parsers/wikiparser/rules/codeinline.js
index ee9149833..048fc051c 100644
--- a/core/modules/parsers/wikiparser/rules/codeinline.js
+++ b/core/modules/parsers/wikiparser/rules/codeinline.js
@@ -33,7 +33,8 @@ exports.parse = function() {
// Look for the end marker
reEnd.lastIndex = this.parser.pos;
var match = reEnd.exec(this.parser.source),
- text;
+ text,
+ start = this.parser.pos;
// Process the text
if(match) {
text = this.parser.source.substring(this.parser.pos,match.index);
@@ -47,7 +48,9 @@ exports.parse = function() {
tag: "code",
children: [{
type: "text",
- text: text
+ text: text,
+ start: start,
+ end: this.parser.pos
}]
}];
};
diff --git a/core/modules/parsers/wikiparser/rules/filteredtranscludeblock.js b/core/modules/parsers/wikiparser/rules/filteredtranscludeblock.js
index 7ab4801bf..73bdff813 100644
--- a/core/modules/parsers/wikiparser/rules/filteredtranscludeblock.js
+++ b/core/modules/parsers/wikiparser/rules/filteredtranscludeblock.js
@@ -31,6 +31,16 @@ exports.init = function(parser) {
exports.parse = function() {
// Move past the match
+ var filterStart = this.parser.pos + 3;
+ var filterEnd = filterStart + this.match[1].length;
+ var toolTipStart = filterEnd + 1;
+ var toolTipEnd = toolTipStart + (this.match[2] ? this.match[2].length : 0);
+ var templateStart = toolTipEnd + 2;
+ var templateEnd = templateStart + (this.match[3] ? this.match[3].length : 0);
+ var styleStart = templateEnd + 2;
+ var styleEnd = styleStart + (this.match[4] ? this.match[4].length : 0);
+ var classesStart = styleEnd + 1;
+ var classesEnd = classesStart + (this.match[5] ? this.match[5].length : 0);
this.parser.pos = this.matchRegExp.lastIndex;
// Get the match details
var filter = this.match[1],
@@ -42,21 +52,21 @@ exports.parse = function() {
var node = {
type: "list",
attributes: {
- filter: {type: "string", value: filter}
+ filter: {type: "string", value: filter, start: filterStart, end: filterEnd},
},
isBlock: true
};
if(tooltip) {
- node.attributes.tooltip = {type: "string", value: tooltip};
+ node.attributes.tooltip = {type: "string", value: tooltip, start: toolTipStart, end: toolTipEnd};
}
if(template) {
- node.attributes.template = {type: "string", value: template};
+ node.attributes.template = {type: "string", value: template, start: templateStart, end: templateEnd};
}
if(style) {
- node.attributes.style = {type: "string", value: style};
+ node.attributes.style = {type: "string", value: style, start: styleStart, end: styleEnd};
}
if(classes) {
- node.attributes.itemClass = {type: "string", value: classes.split(".").join(" ")};
+ node.attributes.itemClass = {type: "string", value: classes.split(".").join(" "), start: classesStart, end: classesEnd};
}
return [node];
};
diff --git a/core/modules/parsers/wikiparser/rules/filteredtranscludeinline.js b/core/modules/parsers/wikiparser/rules/filteredtranscludeinline.js
index 029fd6802..c0b19a941 100644
--- a/core/modules/parsers/wikiparser/rules/filteredtranscludeinline.js
+++ b/core/modules/parsers/wikiparser/rules/filteredtranscludeinline.js
@@ -30,6 +30,16 @@ exports.init = function(parser) {
};
exports.parse = function() {
+ var filterStart = this.parser.pos + 3;
+ var filterEnd = filterStart + this.match[1].length;
+ var toolTipStart = filterEnd + 1;
+ var toolTipEnd = toolTipStart + (this.match[2] ? this.match[2].length : 0);
+ var templateStart = toolTipEnd + 2;
+ var templateEnd = templateStart + (this.match[3] ? this.match[3].length : 0);
+ var styleStart = templateEnd + 2;
+ var styleEnd = styleStart + (this.match[4] ? this.match[4].length : 0);
+ var classesStart = styleEnd + 1;
+ var classesEnd = classesStart + (this.match[5] ? this.match[5].length : 0);
// Move past the match
this.parser.pos = this.matchRegExp.lastIndex;
// Get the match details
@@ -42,20 +52,20 @@ exports.parse = function() {
var node = {
type: "list",
attributes: {
- filter: {type: "string", value: filter}
+ filter: {type: "string", value: filter, start: filterStart, end: filterEnd},
}
};
if(tooltip) {
- node.attributes.tooltip = {type: "string", value: tooltip};
+ node.attributes.tooltip = {type: "string", value: tooltip, start: toolTipStart, end: toolTipEnd};
}
if(template) {
- node.attributes.template = {type: "string", value: template};
+ node.attributes.template = {type: "string", value: template, start: templateStart, end: templateEnd};
}
if(style) {
- node.attributes.style = {type: "string", value: style};
+ node.attributes.style = {type: "string", value: style, start: styleStart, end: styleEnd};
}
if(classes) {
- node.attributes.itemClass = {type: "string", value: classes.split(".").join(" ")};
+ node.attributes.itemClass = {type: "string", value: classes.split(".").join(" "), start: classesStart, end: classesEnd};
}
return [node];
};
diff --git a/core/modules/parsers/wikiparser/rules/hardlinebreaks.js b/core/modules/parsers/wikiparser/rules/hardlinebreaks.js
index c278686b4..94f517cd4 100644
--- a/core/modules/parsers/wikiparser/rules/hardlinebreaks.js
+++ b/core/modules/parsers/wikiparser/rules/hardlinebreaks.js
@@ -45,10 +45,11 @@ exports.parse = function() {
reEnd.lastIndex = this.parser.pos;
match = reEnd.exec(this.parser.source);
if(match) {
+ var start = this.parser.pos;
this.parser.pos = reEnd.lastIndex;
// Add a line break if the terminator was a line break
if(match[2]) {
- tree.push({type: "element", tag: "br"});
+ tree.push({type: "element", tag: "br", start: start, end: this.parser.pos});
}
}
} while(match && !match[1]);
diff --git a/core/modules/parsers/wikiparser/rules/heading.js b/core/modules/parsers/wikiparser/rules/heading.js
index de4e45c27..7a0ecb9db 100644
--- a/core/modules/parsers/wikiparser/rules/heading.js
+++ b/core/modules/parsers/wikiparser/rules/heading.js
@@ -30,15 +30,17 @@ exports.parse = function() {
// Move past the !s
this.parser.pos = this.matchRegExp.lastIndex;
// Parse any classes, whitespace and then the heading itself
+ var classStart = this.parser.pos;
var classes = this.parser.parseClasses();
+ var classEnd = this.parser.pos;
this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
var tree = this.parser.parseInlineRun(/(\r?\n)/mg);
// Return the heading
return [{
type: "element",
- tag: "h" + headingLevel,
+ tag: "h" + headingLevel,
attributes: {
- "class": {type: "string", value: classes.join(" ")}
+ "class": {type: "string", value: classes.join(" "), start: classStart, end: classEnd}
},
children: tree
}];
diff --git a/core/modules/parsers/wikiparser/rules/image.js b/core/modules/parsers/wikiparser/rules/image.js
index 6b379d9c5..6f58225e0 100644
--- a/core/modules/parsers/wikiparser/rules/image.js
+++ b/core/modules/parsers/wikiparser/rules/image.js
@@ -122,9 +122,9 @@ exports.parseImage = function(source,pos) {
}
pos = token.end;
if(token.match[1]) {
- node.attributes.tooltip = {type: "string", value: token.match[1].trim()};
+ node.attributes.tooltip = {type: "string", value: token.match[1].trim(),start: token.start,end:token.start + token.match[1].length - 1};
}
- node.attributes.source = {type: "string", value: (token.match[2] || "").trim()};
+ node.attributes.source = {type: "string", value: (token.match[2] || "").trim(), start: token.start + (token.match[1] ? token.match[1].length : 0), end: token.end - 2};
// Update the end position
node.end = pos;
return node;
diff --git a/core/modules/parsers/wikiparser/rules/import.js b/core/modules/parsers/wikiparser/rules/import.js
index a66df7057..bb1832255 100644
--- a/core/modules/parsers/wikiparser/rules/import.js
+++ b/core/modules/parsers/wikiparser/rules/import.js
@@ -38,13 +38,14 @@ exports.parse = function() {
// Parse the filter terminated by a line break
var reMatch = /(.*)(?:$|\r?\n)/mg;
reMatch.lastIndex = this.parser.pos;
+ var filterStart = this.parser.source;
var match = reMatch.exec(this.parser.source);
this.parser.pos = reMatch.lastIndex;
// Parse tree nodes to return
return [{
type: "importvariables",
attributes: {
- filter: {type: "string", value: match[1]}
+ filter: {type: "string", value: match[1], start: filterStart, end: this.parser.pos}
},
children: []
}];
diff --git a/core/modules/parsers/wikiparser/rules/list.js b/core/modules/parsers/wikiparser/rules/list.js
index d60534403..d89c201b9 100644
--- a/core/modules/parsers/wikiparser/rules/list.js
+++ b/core/modules/parsers/wikiparser/rules/list.js
@@ -131,9 +131,11 @@ exports.parse = function() {
listStack.splice(match[0].length,listStack.length - match[0].length);
}
// Process the body of the list item into the last list item
+ var classStart = this.parser.pos;
var lastListChildren = listStack[listStack.length-1].children,
lastListItem = lastListChildren[lastListChildren.length-1],
classes = this.parser.parseClasses();
+ var classEnd = this.parser.pos;
this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
var tree = this.parser.parseInlineRun(/(\r?\n)/mg);
lastListItem.children.push.apply(lastListItem.children,tree);
@@ -141,6 +143,8 @@ exports.parse = function() {
listStack[listStack.length-1].end = this.parser.pos;
if(classes.length > 0) {
$tw.utils.addClassToParseTreeNode(lastListItem,classes.join(" "));
+ lastListItem.attributes.class.start = classStart;
+ lastListItem.attributes.class.end = classEnd;
}
// Consume any whitespace following the list item
this.parser.skipWhitespace();
diff --git a/core/modules/parsers/wikiparser/rules/prettyextlink.js b/core/modules/parsers/wikiparser/rules/prettyextlink.js
index 9c373cdc9..4707fa0d0 100644
--- a/core/modules/parsers/wikiparser/rules/prettyextlink.js
+++ b/core/modules/parsers/wikiparser/rules/prettyextlink.js
@@ -96,18 +96,20 @@ exports.parseLink = function(source,pos) {
splitPos = null;
}
// Pull out the tooltip and URL
- var tooltip, URL;
+ var tooltip, URL, urlStart;
textNode.start = pos;
if(splitPos) {
+ urlStart = splitPos + 1;
URL = source.substring(splitPos + 1,closePos).trim();
textNode.text = source.substring(pos,splitPos).trim();
textNode.end = splitPos;
} else {
+ urlStart = pos;
URL = source.substring(pos,closePos).trim();
textNode.text = URL;
textNode.end = closePos;
}
- node.attributes.href = {type: "string", value: URL};
+ node.attributes.href = {type: "string", value: URL, start: urlStart, end: closePos};
node.attributes.target = {type: "string", value: "_blank"};
node.attributes.rel = {type: "string", value: "noopener noreferrer"};
// Update the end position
diff --git a/core/modules/parsers/wikiparser/rules/prettylink.js b/core/modules/parsers/wikiparser/rules/prettylink.js
index 5363975af..66c19dc88 100644
--- a/core/modules/parsers/wikiparser/rules/prettylink.js
+++ b/core/modules/parsers/wikiparser/rules/prettylink.js
@@ -29,7 +29,7 @@ exports.init = function(parser) {
exports.parse = function() {
// Move past the match
- var start = this.parser.pos;
+ var start = this.parser.pos + 2;
this.parser.pos = this.matchRegExp.lastIndex;
// Process the link
var text = this.match[1],
@@ -38,12 +38,14 @@ exports.parse = function() {
if (textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
textEndPos = this.matchRegExp.lastIndex - 2;
}
+ var linkStart = this.match[2] ? (start + this.match[1].length + 1) : start;
+ var linkEnd = linkStart + link.length;
if($tw.utils.isLinkExternal(link)) {
return [{
type: "element",
tag: "a",
attributes: {
- href: {type: "string", value: link},
+ href: {type: "string", value: link, start: linkStart, end: linkEnd},
"class": {type: "string", value: "tc-tiddlylink-external"},
target: {type: "string", value: "_blank"},
rel: {type: "string", value: "noopener noreferrer"}
@@ -56,7 +58,7 @@ exports.parse = function() {
return [{
type: "link",
attributes: {
- to: {type: "string", value: link}
+ to: {type: "string", value: link, start: linkStart, end: linkEnd}
},
children: [{
type: "text", text: text, start: start, end: textEndPos
diff --git a/core/modules/parsers/wikiparser/rules/quoteblock.js b/core/modules/parsers/wikiparser/rules/quoteblock.js
index 24b68d24b..aaee84754 100644
--- a/core/modules/parsers/wikiparser/rules/quoteblock.js
+++ b/core/modules/parsers/wikiparser/rules/quoteblock.js
@@ -52,7 +52,9 @@ exports.parse = function() {
this.parser.pos = this.matchRegExp.lastIndex;
// Parse any classes, whitespace and then the optional cite itself
+ var classStart = this.parser.pos;
classes.push.apply(classes, this.parser.parseClasses());
+ var classEnd = this.parser.pos;
this.parser.skipWhitespace({treatNewlinesAsNonWhitespace: true});
var citeStart = this.parser.pos;
var cite = this.parser.parseInlineRun(/(\r?\n)/mg);
@@ -89,7 +91,7 @@ exports.parse = function() {
type: "element",
tag: "blockquote",
attributes: {
- class: { type: "string", value: classes.join(" ") },
+ class: { type: "string", value: classes.join(" "), start: classStart, end: classEnd },
},
children: tree
}];
diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js
index e12eef051..d112f125b 100644
--- a/core/modules/parsers/wikiparser/wikiparser.js
+++ b/core/modules/parsers/wikiparser/wikiparser.js
@@ -91,6 +91,11 @@ var WikiParser = function(type,text,options) {
} else {
topBranch.push.apply(topBranch,this.parseBlocks());
}
+ // Build rules' name map
+ this.usingRuleMap = {};
+ $tw.utils.each(this.pragmaRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
+ $tw.utils.each(this.blockRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
+ $tw.utils.each(this.inlineRules, function (ruleInfo) { self.usingRuleMap[ruleInfo.rule.name] = Object.getPrototypeOf(ruleInfo.rule); });
// Return the parse tree
};
@@ -215,8 +220,7 @@ WikiParser.prototype.parsePragmas = function() {
// Set the start and end positions of the pragma rule if
if (subTree[0].start === undefined) subTree[0].start = start;
if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
- var rule = Object.getPrototypeOf(nextMatch.rule);
- for (const node of subTree) node.rule = rule;
+ $tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
// Quick hack; we only cope with a single parse tree node being returned, which is true at the moment
currentTreeBranch.push.apply(currentTreeBranch,subTree);
subTree[0].children = [];
@@ -246,15 +250,14 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
if (subTree[0].start === undefined) subTree[0].start = start;
if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
}
- var rule = Object.getPrototypeOf(nextMatch.rule);
- for (const node of subTree) node.rule = rule;
+ $tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
return subTree;
}
// Treat it as a paragraph if we didn't find a block rule
var start = this.pos;
var children = this.parseInlineRun(terminatorRegExp);
var end = this.pos;
- return [{type: "element", tag: "p", children: children, start: start, end: end, parseRule: undefined }];
+ return [{type: "element", tag: "p", children: children, start: start, end: end, rule: null }];
};
/*
@@ -353,8 +356,7 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
if (subTree[0].start === undefined) subTree[0].start = start;
if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
}
- var rule = Object.getPrototypeOf(nextMatch.rule);
- $tw.utils.each(subTree, function (node) { node.rule = rule; });
+ $tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
tree.push.apply(tree,subTree);
// Look for the next run rule
nextMatch = this.findNextMatch(this.inlineRules,this.pos);
@@ -413,8 +415,7 @@ WikiParser.prototype.parseInlineRunTerminatedExtended = function(terminatorRegEx
if (subTree[0].start === undefined) subTree[0].start = start;
if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
}
- var rule = Object.getPrototypeOf(inlineRuleMatch.rule);
- for (const node of subTree) node.rule = rule;
+ $tw.utils.each(subTree, function (node) { node.rule = inlineRuleMatch.rule.name; });
tree.push.apply(tree,subTree);
// Look for the next inline rule
inlineRuleMatch = this.findNextMatch(this.inlineRules,this.pos);
@@ -441,7 +442,7 @@ WikiParser.prototype.pushTextWidget = function(array,text,start,end) {
text = $tw.utils.trim(text);
}
if(text) {
- array.push({type: "text", text: text, start: start, end: end, parseRule: undefined});
+ array.push({type: "text", text: text, start: start, end: end, rule: null});
}
};
diff --git a/editions/test/tiddlers/tests/test-wikitext-parser.js b/editions/test/tiddlers/tests/test-wikitext-parser.js
index bc3d9acd8..3e7f1ff91 100644
--- a/editions/test/tiddlers/tests/test-wikitext-parser.js
+++ b/editions/test/tiddlers/tests/test-wikitext-parser.js
@@ -26,88 +26,87 @@ describe("WikiText parser tests", function() {
it("should parse tags", function() {
expect(parse("
")).toEqual(
- [ { type : 'element', tag : 'p', start : 0, end : 4, children : [ { type : 'element', tag : 'br', start : 0, end : 4, isBlock : false, attributes : { }, orderedAttributes: [ ] } ] } ]
+ [ { type : 'element', tag : 'p', start : 0, end : 4, rule: null, children : [ { type : 'element', tag : 'br', start : 0, end : 4, rule: 'html', isBlock : false, attributes : { }, orderedAttributes: [ ] } ] } ]
);
expect(parse("")).toEqual(
- [ { type : 'element', tag : 'p', start : 0, end : 5, children : [ { type : 'text', text : '', start : 0, end : 5 } ] } ]
+ [ { type : 'element', tag : 'p', start : 0, end : 5, rule: null, children : [ { type : 'text', text : '', start : 0, end : 5, rule: null } ] } ]
);
expect(parse("