From 2bd9cc45fa77bc6b45a99a5ca4da1ed4d8e7b03d Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Wed, 4 Nov 2020 21:02:17 +0100 Subject: [PATCH 1/5] Added more start/end parser ranges --- core/modules/parsers/wikiparser/wikiparser.js | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js index 673176038..8d1d28aee 100644 --- a/core/modules/parsers/wikiparser/wikiparser.js +++ b/core/modules/parsers/wikiparser/wikiparser.js @@ -165,6 +165,17 @@ WikiParser.prototype.findNextMatch = function(rules,startPos) { return matchingRule; }; +WikiParser.prototype.parseRule = function(rule) { + var start = this.pos; + var blocks = rule.parse(); + for(var i=0; i 0) { // Quick hack; we only cope with a single parse tree node being returned, which is true at the moment currentTreeBranch.push.apply(currentTreeBranch,subTree); @@ -208,10 +219,13 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) { // Look for a block rule that applies at the current position var nextMatch = this.findNextMatch(this.blockRules,this.pos); if(nextMatch && nextMatch.matchIndex === this.pos) { - return nextMatch.rule.parse(); + return this.parseRule(nextMatch.rule); } // Treat it as a paragraph if we didn't find a block rule - return [{type: "element", tag: "p", children: this.parseInlineRun(terminatorRegExp)}]; + var start = this.pos; + var children = this.parseInlineRun(terminatorRegExp); + var end = this.pos; + return [{type: "element", tag: "p", children, start, end }]; }; /* @@ -287,17 +301,17 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) { while(this.pos < this.sourceLength && nextMatch) { // Process the text preceding the run rule if(nextMatch.matchIndex > this.pos) { - this.pushTextWidget(tree,this.source.substring(this.pos,nextMatch.matchIndex)); + this.pushTextWidget(tree,this.source.substring(this.pos,nextMatch.matchIndex),this.pos,nextMatch.matchIndex); this.pos = nextMatch.matchIndex; } // Process the run rule - tree.push.apply(tree,nextMatch.rule.parse()); + tree.push.apply(tree,this.parseRule(nextMatch.rule)); // Look for the next run rule nextMatch = this.findNextMatch(this.inlineRules,this.pos); } // Process the remaining text if(this.pos < this.sourceLength) { - this.pushTextWidget(tree,this.source.substr(this.pos)); + this.pushTextWidget(tree,this.source.substr(this.pos),this.pos,this.sourceLength); } this.pos = this.sourceLength; return tree; @@ -317,7 +331,7 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option if(terminatorMatch) { if(!inlineRuleMatch || inlineRuleMatch.matchIndex >= terminatorMatch.index) { if(terminatorMatch.index > this.pos) { - this.pushTextWidget(tree,this.source.substring(this.pos,terminatorMatch.index)); + this.pushTextWidget(tree,this.source.substring(this.pos,terminatorMatch.index),this.pos,terminatorMatch.index); } this.pos = terminatorMatch.index; if(options.eatTerminator) { @@ -330,11 +344,11 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option if(inlineRuleMatch) { // Preceding text if(inlineRuleMatch.matchIndex > this.pos) { - this.pushTextWidget(tree,this.source.substring(this.pos,inlineRuleMatch.matchIndex)); + this.pushTextWidget(tree,this.source.substring(this.pos,inlineRuleMatch.matchIndex),this.pos,inlineRuleMatch.matchIndex); this.pos = inlineRuleMatch.matchIndex; } // Process the inline rule - tree.push.apply(tree,inlineRuleMatch.rule.parse()); + tree.push.apply(tree,this.parseRule(inlineRuleMatch.rule)); // Look for the next inline rule inlineRuleMatch = this.findNextMatch(this.inlineRules,this.pos); // Look for the next terminator match @@ -344,7 +358,7 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option } // Process the remaining text if(this.pos < this.sourceLength) { - this.pushTextWidget(tree,this.source.substr(this.pos)); + this.pushTextWidget(tree,this.source.substr(this.pos),this.pos,this.sourceLength); } this.pos = this.sourceLength; return tree; @@ -353,12 +367,12 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option /* Push a text widget onto an array, respecting the configTrimWhiteSpace setting */ -WikiParser.prototype.pushTextWidget = function(array,text) { +WikiParser.prototype.pushTextWidget = function(array,text,start,end) { if(this.configTrimWhiteSpace) { text = $tw.utils.trim(text); } if(text) { - array.push({type: "text", text: text}); + array.push({type: "text", text: text, start: start, end: end}); } }; From 1b226c7556d08d46d2e2d94e2b9b338d8441f3e7 Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Fri, 6 Nov 2020 19:56:20 +0100 Subject: [PATCH 2/5] Fixed coding standard nits --- core/modules/parsers/wikiparser/wikiparser.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js index 8d1d28aee..9af8ddff0 100644 --- a/core/modules/parsers/wikiparser/wikiparser.js +++ b/core/modules/parsers/wikiparser/wikiparser.js @@ -166,12 +166,16 @@ WikiParser.prototype.findNextMatch = function(rules,startPos) { }; WikiParser.prototype.parseRule = function(rule) { - var start = this.pos; - var blocks = rule.parse(); + var start = this.pos, + blocks = rule.parse(); for(var i=0; i Date: Sun, 25 Apr 2021 13:12:45 +0200 Subject: [PATCH 3/5] Replaced restructuring and added fallback comment --- core/modules/parsers/wikiparser/wikiparser.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js index 9af8ddff0..922bb541f 100644 --- a/core/modules/parsers/wikiparser/wikiparser.js +++ b/core/modules/parsers/wikiparser/wikiparser.js @@ -168,6 +168,7 @@ WikiParser.prototype.findNextMatch = function(rules,startPos) { WikiParser.prototype.parseRule = function(rule) { var start = this.pos, blocks = rule.parse(); + // Use fallback parent range for blocks that don't define their own for(var i=0; i Date: Sun, 25 Apr 2021 16:03:35 +0200 Subject: [PATCH 4/5] Fixed fallback range logic + added ranges to tests --- core/modules/parsers/wikiparser/wikiparser.js | 24 +++++-- .../tiddlers/tests/test-wikitext-parser.js | 62 +++++++++---------- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js index 606b5f6f8..6416d14e1 100644 --- a/core/modules/parsers/wikiparser/wikiparser.js +++ b/core/modules/parsers/wikiparser/wikiparser.js @@ -187,20 +187,34 @@ WikiParser.prototype.findNextMatch = function(rules,startPos) { WikiParser.prototype.parseRule = function(rule) { var start = this.pos, - blocks = rule.parse(); - // Use fallback parent range for blocks that don't define their own + blocks = rule.parse(), + pending = []; + // Estimate start/end ranges for blocks that don't define their own based on + // sibling and parent ranges for(var i=0; i")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'br', isBlock : false, attributes : { }, start : 0, end : 4 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 4, children : [ { type : 'element', tag : 'br', start : 0, end : 4, isBlock : false, attributes : { } } ] } ] ); expect(parse("
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'text', text : '
' } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 5, children : [ { type : 'text', text : '
', start : 0, end : 5 } ] } ] ); expect(parse("
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { }, children : [ ], start : 0, end : 5 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 5, children : [ { type : 'element', tag : 'div', start : 0, end : 5, isBlock : false, attributes : { }, children : [ ] } ] } ] ); expect(parse("
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isSelfClosing : true, isBlock : false, attributes : { }, start : 0, end : 6 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 6, children : [ { type : 'element', tag : 'div', isSelfClosing : true, isBlock : false, attributes : { }, start : 0, end : 6 } ] } ] ); expect(parse("
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { }, children : [ ], start : 0, end : 5 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 11, children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { }, children : [ ], start : 0, end : 5 } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 5 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 20, children : [ { type : 'element', tag : 'div', start : 0, end : 20, isBlock : false, attributes : { }, children : [ { type : 'text', text : 'some text', start : 5, end : 14 } ], start : 0, end : 5 } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', value : 'true', start : 4, end : 14, name: 'attribute' } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 15 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 30, children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', value : 'true', start : 4, end : 14, name: 'attribute' } }, children : [ { type : 'text', text : 'some text', start : 15, end : 24 } ], start : 0, end : 15 } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'string', name: 'attribute', value : 'value', start: 4, end: 22 } }, children : [ { type : 'text', text : 'some text' } ], start: 0, end: 23 } ] } ] + [ { type : 'element', tag : 'p', start : 0, end : 38, children : [ { type : 'element', tag : 'div', start: 0, end: 38, isBlock : false, attributes : { attribute : { type : 'string', name: 'attribute', value : 'value', start: 4, end: 22 } }, children : [ { type : 'text', text : 'some text', start : 23, end : 32 } ], start : 0, end : 23 } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name: 'attribute', textReference : 'TiddlerTitle', start : 4, end : 31 } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 32 } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 47, children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name: 'attribute', textReference : 'TiddlerTitle', start : 4, end : 31 } }, children : [ { type : 'text', text : 'some text', start : 32, end : 41 } ], start : 0, end : 32 } ] } ] ); expect(parse("<$reveal state='$:/temp/search' type='nomatch' text=''>")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'reveal', tag: '$reveal', start : 0, attributes : { state : { start : 8, name : 'state', type : 'string', value : '$:/temp/search', end : 31 }, type : { start : 31, name : 'type', type : 'string', value : 'nomatch', end : 46 }, text : { start : 46, name : 'text', type : 'string', value : '', end : 54 } }, end : 55, isBlock : false, children : [ ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 55, children : [ { type : 'reveal', tag: '$reveal', start : 0, attributes : { state : { start : 8, name : 'state', type : 'string', value : '$:/temp/search', end : 31 }, type : { start : 31, name : 'type', type : 'string', value : 'nomatch', end : 46 }, text : { start : 46, name : 'text', type : 'string', value : '', end : 54 } }, end : 55, isBlock : false, children : [ ] } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name : 'attribute', textReference : 'TiddlerTitle!!field', start : 4, end : 38 } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 39 } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 54, children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name : 'attribute', textReference : 'TiddlerTitle!!field', start : 4, end : 38 } }, children : [ { type : 'text', text : 'some text', start : 39, end : 48 } ], start : 0, end : 39 } ] } ] ); expect(parse("
some text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name : 'attribute', textReference : 'Tiddler Title!!field', start : 4, end : 39 } }, children : [ { type : 'text', text : 'some text' } ], start : 0, end : 40 } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 55, children : [ { type : 'element', tag : 'div', isBlock : false, attributes : { attribute : { type : 'indirect', name : 'attribute', textReference : 'Tiddler Title!!field', start : 4, end : 39 } }, children : [ { type : 'text', text : 'some text', start : 40, end : 49 } ], start : 0, end : 40 } ] } ] ); expect(parse("
\n\nsome text
")).toEqual( - [ { type : 'element', start : 0, attributes : { attribute : { start : 4, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 38 } }, tag : 'div', end : 39, isBlock : true, children : [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'some text' } ] } ] } ] + [ { type : 'element', start : 0, attributes : { attribute : { start : 4, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 38 } }, tag : 'div', end : 39, isBlock : true, children : [ { type : 'element', tag : 'p', start : 41, end : 50, children : [ { type : 'text', text : 'some text', start : 41, end : 50 } ] } ] } ] ); expect(parse("
\n\nsome text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'some text' } ] } ] } ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 67, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'p', start : 46, end : 55, children : [ { type : 'text', text : 'some text', start : 46, end : 55 } ] } ] } ] } ] } ] ); expect(parse("
\n\n!some heading
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'h1', attributes : { class : { type : 'string', value : '' } }, children : [ { type : 'text', text : 'some heading
' } ] } ] } ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 71, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'h1', start : 46, end : 71, attributes : { class : { type : 'string', value : '' } }, children : [ { type : 'text', text : 'some heading', start : 47, end : 71 } ] } ] } ] } ] } ] ); expect(parse("
\n!some heading
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : false, children : [ { type : 'text', text : '\n!some heading' } ] } ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 70, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : false, children : [ { type : 'text', text : '\n!some heading', start : 44, end : 58 } ] } ] } ] } ] ); // Regression test for issue (#3306) expect(parse("
\n\nSome text
")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { }, tag : 'span', end : 11, isBlock : false, children : [ { type : 'element', start : 11, attributes : { }, tag : 'span', end : 17, isBlock : true, children : [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Some text' } ] } ] } ] } ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 48, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { }, tag : 'span', end : 11, isBlock : false, children : [ { type : 'element', start : 11, attributes : { }, tag : 'span', end : 17, isBlock : true, children : [ { type : 'element', tag : 'p', start : 19, end : 28, children : [ { type : 'text', text : 'Some text', start : 19, end : 28 } ] } ] } ] } ] } ] } ] ); }); @@ -114,7 +114,7 @@ describe("WikiText parser tests", function() { it("should parse macro definitions", function() { expect(parse("\\define myMacro()\nnothing\n\\end\n")).toEqual( - [ { type : 'set', attributes : { name : { type : 'string', value : 'myMacro' }, value : { type : 'string', value : 'nothing' } }, children : [ ], params : [ ], isMacroDefinition : true } ] + [ { type : 'set', start : 0, end : 30, attributes : { name : { type : 'string', value : 'myMacro' }, value : { type : 'string', value : 'nothing' } }, children : [ ], params : [ ], isMacroDefinition : true } ] ); @@ -123,38 +123,38 @@ describe("WikiText parser tests", function() { it("should parse inline macro calls", function() { expect(parse("<><><><>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'macrocall', start: 0, params: [ ], name: 'john', end: 8 }, { type: 'macrocall', start: 8, params: [ ], name: 'paul', end: 16 }, { type: 'macrocall', start: 16, params: [ ], name: 'george', end: 26 }, { type: 'macrocall', start: 26, params: [ ], name: 'ringo', end: 35 } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 35, children: [ { type: 'macrocall', start: 0, params: [ ], name: 'john', end: 8 }, { type: 'macrocall', start: 8, params: [ ], name: 'paul', end: 16 }, { type: 'macrocall', start: 16, params: [ ], name: 'george', end: 26 }, { type: 'macrocall', start: 26, params: [ ], name: 'ringo', end: 35 } ] } ] ); expect(parse("text <>")).toEqual( - [{ type: 'element', tag: 'p', children: [ { type: 'text', text: 'text ' }, { type: 'macrocall', name: 'john', start: 5, params: [ { type: 'macro-parameter', start: 11, value: 'val1', name: 'one', end: 20 }, { type: 'macro-parameter', start: 20, value: 'val "2"', name: 'two', end: 35 }, { type: 'macro-parameter', start: 35, value: 'val \'3\'', name: 'three', end: 52 }, { type: 'macro-parameter', start: 52, value: 'val 4"5\'', name: 'four', end: 73 }, { type: 'macro-parameter', start: 73, value: 'val 5', name: 'five', end: 89 } ], end: 92 } ] } ] + [{ type: 'element', tag: 'p', start: 0, end: 92, children: [ { type: 'text', text: 'text ', start: 0, end: 5 }, { type: 'macrocall', name: 'john', start: 5, params: [ { type: 'macro-parameter', start: 11, value: 'val1', name: 'one', end: 20 }, { type: 'macro-parameter', start: 20, value: 'val "2"', name: 'two', end: 35 }, { type: 'macro-parameter', start: 35, value: 'val \'3\'', name: 'three', end: 52 }, { type: 'macro-parameter', start: 52, value: 'val 4"5\'', name: 'four', end: 73 }, { type: 'macro-parameter', start: 73, value: 'val 5', name: 'five', end: 89 } ], end: 92 } ] } ] ); expect(parse("ignored << carrots <>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: 'ignored << carrots ' }, { type: 'macrocall', name: 'john', start: 19, params: [ ], end: 27 } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 27, children: [ { type: 'text', text: 'ignored << carrots ', start: 0, end: 19 }, { type: 'macrocall', name: 'john', start: 19, params: [ ], end: 27 } ] } ] ); expect(parse("text <<>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: 'text ' }, { type: 'macrocall', name: '>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: 'before\n' }, { type: 'macrocall', start: 7, params: [ ], name: 'john', end: 15 } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 15, children: [ { type: 'text', text: 'before\n', start: 0, end: 7 }, { type: 'macrocall', start: 7, params: [ ], name: 'john', end: 15 } ] } ] ); // A single space will cause it to be inline expect(parse("<> ")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'macrocall', start: 0, params: [ ], name: 'john', end: 8 }, { type: 'text', text: ' ' } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 9, children: [ { type: 'macrocall', start: 0, params: [ ], name: 'john', end: 8 }, { type: 'text', text: ' ', start: 8, end: 9 } ] } ] ); expect(parse("text <>' >>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: 'text ' }, { type: 'macrocall', start: 5, params: [ { type: 'macro-parameter', start: 12, value: 'my <>', name: 'one', end: 31 } ], name: 'outie', end: 34 } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 34, children: [ { type: 'text', text: 'text ', start: 0, end: 5 }, { type: 'macrocall', start: 5, params: [ { type: 'macro-parameter', start: 12, value: 'my <>', name: 'one', end: 31 } ], name: 'outie', end: 34 } ] } ] ); @@ -173,17 +173,17 @@ describe("WikiText parser tests", function() { ); expect(parse("<< carrots\n\n<>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: '<< carrots' } ] }, { type: 'macrocall', start: 12, params: [ ], name: 'john', end: 20, isBlock: true } ] + [ { type: 'element', tag: 'p', start : 0, end : 10, children: [ { type: 'text', text: '<< carrots', start : 0, end : 10 } ] }, { type: 'macrocall', start: 12, params: [ ], name: 'john', end: 20, isBlock: true } ] ); expect(parse("before\n\n<>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'text', text: 'before' } ] }, { type: 'macrocall', start: 8, name: 'john', params: [ ], end: 16, isBlock: true } ] + [ { type: 'element', tag: 'p', start : 0, end : 6, children: [ { type: 'text', text: 'before', start : 0, end : 6 } ] }, { type: 'macrocall', start: 8, name: 'john', params: [ ], end: 16, isBlock: true } ] ); expect(parse("<>\nafter")).toEqual( - [ { type: 'macrocall', start: 0, name: 'john', params: [ ], end: 8, isBlock: true }, { type: 'element', tag: 'p', children: [ { type: 'text', text: 'after' } ] } ] + [ { type: 'macrocall', start: 0, name: 'john', params: [ ], end: 8, isBlock: true }, { type: 'element', tag: 'p', start: 9, end: 14, children: [ { type: 'text', text: 'after', start: 9, end: 14 } ] } ] ); expect(parse("<>")).toEqual( @@ -211,7 +211,7 @@ describe("WikiText parser tests", function() { ); expect(parse("<>>")).toEqual( - [ { type: 'element', tag: 'p', children: [ { type: 'macrocall', start: 0, params: [ { type: 'macro-parameter', start: 6, value: 'param', end: 12 } ], name: 'john', end: 14 }, { type: 'text', text: '>' } ] } ] + [ { type: 'element', tag: 'p', start: 0, end: 15, children: [ { type: 'macrocall', start: 0, params: [ { type: 'macro-parameter', start: 6, value: 'param', end: 12 } ], name: 'john', end: 14 }, { type: 'text', text: '>', start: 14, end: 15 } ] } ] ); // equals signs should be allowed @@ -226,7 +226,7 @@ describe("WikiText parser tests", function() { it("should parse horizontal rules", function() { expect(parse("---Not a rule\n\n----\n\nBetween\n\n---")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'entity', entity : '—' }, { type : 'text', text : 'Not a rule' } ] }, { type : 'element', tag : 'hr' }, { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Between' } ] }, { type : 'element', tag : 'hr' } ] + [ { type : 'element', tag : 'p', start : 0, end : 13, children : [ { type : 'entity', entity : '—', start : 0, end : 3 }, { type : 'text', text : 'Not a rule', start : 3, end : 13 } ] }, { type : 'element', tag : 'hr', start : 15, end : 20 }, { type : 'element', tag : 'p', start : 21, end : 28, children : [ { type : 'text', text : 'Between', start : 21, end : 28 } ] }, { type : 'element', tag : 'hr', start : 30, end : 33 } ] ); @@ -235,7 +235,7 @@ describe("WikiText parser tests", function() { it("should parse hard linebreak areas", function() { expect(parse("\"\"\"Something\nin the\nway she moves\n\"\"\"\n\n")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Something' }, { type : 'element', tag : 'br' }, { type : 'text', text : 'in the' }, { type : 'element', tag : 'br' }, { type : 'text', text : 'way she moves' }, { type : 'element', tag : 'br' } ] } ] + [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Something', start : 3, end : 12 }, { type : 'element', tag : 'br', start : 12, end : 13 }, { type : 'text', text : 'in the', start : 13, end : 19 }, { type : 'element', tag : 'br', start : 19, end : 20 }, { type : 'text', text : 'way she moves', start : 20, end : 33 }, { type : 'element', tag : 'br', start: 33, end: 37 } ], start : 0, end : 37 } ] ); From 9b247f6d63051877781d909ecab5e2e373555acb Mon Sep 17 00:00:00 2001 From: Miha Lunar Date: Sat, 29 May 2021 20:30:04 +0200 Subject: [PATCH 5/5] Removed fallback range logic --- core/modules/parsers/wikiparser/wikiparser.js | 38 ++----------------- .../tiddlers/tests/test-wikitext-parser.js | 8 ++-- 2 files changed, 8 insertions(+), 38 deletions(-) diff --git a/core/modules/parsers/wikiparser/wikiparser.js b/core/modules/parsers/wikiparser/wikiparser.js index 6416d14e1..90a3e7446 100644 --- a/core/modules/parsers/wikiparser/wikiparser.js +++ b/core/modules/parsers/wikiparser/wikiparser.js @@ -185,36 +185,6 @@ WikiParser.prototype.findNextMatch = function(rules,startPos) { return matchingRule; }; -WikiParser.prototype.parseRule = function(rule) { - var start = this.pos, - blocks = rule.parse(), - pending = []; - // Estimate start/end ranges for blocks that don't define their own based on - // sibling and parent ranges - for(var i=0; i 0) { // Quick hack; we only cope with a single parse tree node being returned, which is true at the moment currentTreeBranch.push.apply(currentTreeBranch,subTree); @@ -258,7 +228,7 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) { // Look for a block rule that applies at the current position var nextMatch = this.findNextMatch(this.blockRules,this.pos); if(nextMatch && nextMatch.matchIndex === this.pos) { - return this.parseRule(nextMatch.rule); + return nextMatch.rule.parse(); } // Treat it as a paragraph if we didn't find a block rule var start = this.pos; @@ -344,7 +314,7 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) { this.pos = nextMatch.matchIndex; } // Process the run rule - tree.push.apply(tree,this.parseRule(nextMatch.rule)); + tree.push.apply(tree,nextMatch.rule.parse()); // Look for the next run rule nextMatch = this.findNextMatch(this.inlineRules,this.pos); } @@ -387,7 +357,7 @@ WikiParser.prototype.parseInlineRunTerminated = function(terminatorRegExp,option this.pos = inlineRuleMatch.matchIndex; } // Process the inline rule - tree.push.apply(tree,this.parseRule(inlineRuleMatch.rule)); + tree.push.apply(tree,inlineRuleMatch.rule.parse()); // Look for the next inline rule inlineRuleMatch = this.findNextMatch(this.inlineRules,this.pos); // Look for the next terminator match diff --git a/editions/test/tiddlers/tests/test-wikitext-parser.js b/editions/test/tiddlers/tests/test-wikitext-parser.js index 041ca31c0..da4965c3a 100644 --- a/editions/test/tiddlers/tests/test-wikitext-parser.js +++ b/editions/test/tiddlers/tests/test-wikitext-parser.js @@ -95,7 +95,7 @@ describe("WikiText parser tests", function() { ); expect(parse("
\n\n!some heading
")).toEqual( - [ { type : 'element', tag : 'p', start: 0, end: 71, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'h1', start : 46, end : 71, attributes : { class : { type : 'string', value : '' } }, children : [ { type : 'text', text : 'some heading', start : 47, end : 71 } ] } ] } ] } ] } ] + [ { type : 'element', tag : 'p', start: 0, end: 71, children : [ { type : 'element', start : 0, attributes : { }, tag : 'div', end : 5, isBlock : false, children : [ { type : 'element', start : 5, attributes : { attribute : { start : 9, name : 'attribute', type : 'indirect', textReference : 'TiddlerTitle!!field', end : 43 } }, tag : 'div', end : 44, isBlock : true, children : [ { type : 'element', tag : 'h1', attributes : { class : { type : 'string', value : '' } }, children : [ { type : 'text', text : 'some heading', start : 47, end : 71 } ] } ] } ] } ] } ] ); expect(parse("
\n!some heading
")).toEqual( @@ -114,7 +114,7 @@ describe("WikiText parser tests", function() { it("should parse macro definitions", function() { expect(parse("\\define myMacro()\nnothing\n\\end\n")).toEqual( - [ { type : 'set', start : 0, end : 30, attributes : { name : { type : 'string', value : 'myMacro' }, value : { type : 'string', value : 'nothing' } }, children : [ ], params : [ ], isMacroDefinition : true } ] + [ { type : 'set', attributes : { name : { type : 'string', value : 'myMacro' }, value : { type : 'string', value : 'nothing' } }, children : [ ], params : [ ], isMacroDefinition : true } ] ); @@ -226,7 +226,7 @@ describe("WikiText parser tests", function() { it("should parse horizontal rules", function() { expect(parse("---Not a rule\n\n----\n\nBetween\n\n---")).toEqual( - [ { type : 'element', tag : 'p', start : 0, end : 13, children : [ { type : 'entity', entity : '—', start : 0, end : 3 }, { type : 'text', text : 'Not a rule', start : 3, end : 13 } ] }, { type : 'element', tag : 'hr', start : 15, end : 20 }, { type : 'element', tag : 'p', start : 21, end : 28, children : [ { type : 'text', text : 'Between', start : 21, end : 28 } ] }, { type : 'element', tag : 'hr', start : 30, end : 33 } ] + [ { type : 'element', tag : 'p', start : 0, end : 13, children : [ { type : 'entity', entity : '—' }, { type : 'text', text : 'Not a rule', start : 3, end : 13 } ] }, { type : 'element', tag : 'hr' }, { type : 'element', tag : 'p', start : 21, end : 28, children : [ { type : 'text', text : 'Between', start : 21, end : 28 } ] }, { type : 'element', tag : 'hr' } ] ); @@ -235,7 +235,7 @@ describe("WikiText parser tests", function() { it("should parse hard linebreak areas", function() { expect(parse("\"\"\"Something\nin the\nway she moves\n\"\"\"\n\n")).toEqual( - [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Something', start : 3, end : 12 }, { type : 'element', tag : 'br', start : 12, end : 13 }, { type : 'text', text : 'in the', start : 13, end : 19 }, { type : 'element', tag : 'br', start : 19, end : 20 }, { type : 'text', text : 'way she moves', start : 20, end : 33 }, { type : 'element', tag : 'br', start: 33, end: 37 } ], start : 0, end : 37 } ] + [ { type : 'element', tag : 'p', children : [ { type : 'text', text : 'Something', start : 3, end : 12 }, { type : 'element', tag : 'br' }, { type : 'text', text : 'in the', start : 13, end : 19 }, { type : 'element', tag : 'br' }, { type : 'text', text : 'way she moves', start : 20, end : 33 }, { type : 'element', tag : 'br' } ], start : 0, end : 37 } ] );