diff --git a/core/modules/parsers/wikiparser/rules/codeinline.js b/core/modules/parsers/wikiparser/rules/codeinline.js index 5606ba478..73e3484aa 100644 --- a/core/modules/parsers/wikiparser/rules/codeinline.js +++ b/core/modules/parsers/wikiparser/rules/codeinline.js @@ -31,13 +31,16 @@ exports.parse = function() { reEnd.lastIndex = this.parser.pos; var match = reEnd.exec(this.parser.source), text, - start = this.parser.pos; + start = this.parser.pos, + textEnd; // Process the text if(match) { text = this.parser.source.substring(this.parser.pos,match.index); + textEnd = match.index; this.parser.pos = match.index + match[0].length; } else { text = this.parser.source.substr(this.parser.pos); + textEnd = this.parser.sourceLength; this.parser.pos = this.parser.sourceLength; } return [{ @@ -47,7 +50,7 @@ exports.parse = function() { type: "text", text: text, start: start, - end: this.parser.pos + end: textEnd }] }]; }; diff --git a/core/modules/parsers/wikiparser/rules/extlink.js b/core/modules/parsers/wikiparser/rules/extlink.js index 7a2485f1c..ca8c13012 100644 --- a/core/modules/parsers/wikiparser/rules/extlink.js +++ b/core/modules/parsers/wikiparser/rules/extlink.js @@ -32,7 +32,8 @@ exports.parse = function() { this.parser.pos = this.matchRegExp.lastIndex; // Create the link unless it is suppressed if(this.match[0].substr(0,1) === "~") { - return [{type: "text", text: this.match[0].substr(1), start: start, end: this.parser.pos}]; + // Start after the suppressing "~" so the span matches the plain text + return [{type: "text", text: this.match[0].substr(1), start: start + 1, end: this.parser.pos}]; } else { return [{ type: "element", diff --git a/core/modules/parsers/wikiparser/rules/import.js b/core/modules/parsers/wikiparser/rules/import.js index 65376f8be..fe291f8e3 100644 --- a/core/modules/parsers/wikiparser/rules/import.js +++ b/core/modules/parsers/wikiparser/rules/import.js @@ -34,7 +34,7 @@ exports.parse = function() { // Parse the filter terminated by a line break var reMatch = /(.*)(?:$|\r?\n)/mg; reMatch.lastIndex = this.parser.pos; - var filterStart = this.parser.source; + var filterStart = this.parser.pos; var match = reMatch.exec(this.parser.source); this.parser.pos = reMatch.lastIndex; // Parse tree nodes to return diff --git a/core/modules/parsers/wikiparser/rules/wikilinkprefix.js b/core/modules/parsers/wikiparser/rules/wikilinkprefix.js index c1c5db1f6..a1f63293e 100644 --- a/core/modules/parsers/wikiparser/rules/wikilinkprefix.js +++ b/core/modules/parsers/wikiparser/rules/wikilinkprefix.js @@ -27,9 +27,11 @@ Parse the most recent match */ exports.parse = function() { // Get the details of the match - var linkText = this.match[0]; + var linkText = this.match[0], + // Start after the suppressing "~" so the span matches the plain text + start = this.parser.pos + 1; // Move past the wikilink this.parser.pos = this.matchRegExp.lastIndex; // Return the link without unwikilink character as plain text - return [{type: "text", text: linkText.substr(1)}]; + return [{type: "text", text: linkText.substr(1), start: start, end: this.parser.pos}]; }; diff --git a/editions/test/tiddlers/tests/test-parsetree-positions.js b/editions/test/tiddlers/tests/test-parsetree-positions.js new file mode 100644 index 000000000..e41dc0648 --- /dev/null +++ b/editions/test/tiddlers/tests/test-parsetree-positions.js @@ -0,0 +1,60 @@ +/*\ +title: test-parsetree-positions.js +type: application/javascript +tags: [[$:/tags/test-spec]] + +Regression tests for #9882: wikitext parser rules must emit accurate +`start`/`end` source positions on their parse tree nodes. Tooling that maps +rendered output back to the source text relies on these offsets. + +\*/ + +"use strict"; + +describe("Parse tree source position tests (#9882)", function() { + + // Create a wiki + var wiki = $tw.test.wiki(); + + // Define a parsing shortcut + var parse = function(text) { + return wiki.parseText("text/vnd.tiddlywiki",text).tree; + }; + + it("should give inline code runs a text node that spans only the code, not the backticks", function() { + // codeinline.js: `code` gives a text node "code" spanning offsets 1 to 5. The closing backtick at 5 is excluded. + // Bug: `end` was set to `this.parser.pos`, which sits past the closing backtick, so end was 6 and the span swallowed the backtick. + expect(parse("`code`")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 6, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 6, children: [ { type: "text", text: "code", start: 1, end: 5 } ] } ] } ] + ); + // ``a`b`` gives text "a`b" spanning 2 to 5. `end` must be the offset of the closing marker whatever its length. + // Bug: `end` was `this.parser.pos` (7), two characters past the code, so it also swallowed the closing ``. + expect(parse("``a`b``")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 7, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 7, children: [ { type: "text", text: "a`b", start: 2, end: 5 } ] } ] } ] + ); + }); + + it("should start the text node of a suppressed external link after the ~", function() { + // extlink.js: ~https://example.com/ emits the plain text "https://example.com/", which spans offsets 1 to 21. + // Bug: `start` was the offset of the ~ (0), so the span was one character too wide and began on the ~ that the text omits. + expect(parse("~https://example.com/")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 21, children: [ { type: "text", text: "https://example.com/", start: 1, end: 21, rule: "extlink" } ] } ] + ); + }); + + it("should give a suppressed wikilink's text node source positions", function() { + // wikilinkprefix.js: ~SuppressedLink emits the plain text "SuppressedLink", spanning offsets 1 to 15. + // Bug: the text node carried no `start`/`end` at all. The parser framework then defaulted `start` to the ~ offset (0). + expect(parse("~SuppressedLink")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 15, children: [ { type: "text", text: "SuppressedLink", start: 1, end: 15, rule: "wikilinkprefix" } ] } ] + ); + }); + + it("should record the filter's start offset for an \\import pragma", function() { + // import.js: \import [tag[x]] records the filter value "[tag[x]]" starting at offset 8, right after "\import ". + // Bug: `filterStart` was assigned `this.parser.source` (the whole source string) instead of `this.parser.pos`, so `start` was a string, not an offset. + expect(parse("\\import [tag[x]]\n")).toEqual( + [ { type: "importvariables", rule: "import", start: 0, end: 16, attributes: { filter: { type: "string", value: "[tag[x]]", start: 8, end: 16 } }, children: [] } ] + ); + }); +}); diff --git a/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid b/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid new file mode 100644 index 000000000..b833ec348 --- /dev/null +++ b/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid @@ -0,0 +1,21 @@ +title: $:/changenotes/5.4.1/#9882 +created: 20260701142414000 +modified: 20260701142414000 +description: Fix invalid source positions in parse tree nodes +release: 5.4.1 +tags: $:/tags/ChangeNote +change-type: bugfix +change-category: internal +github-links: https://github.com/TiddlyWiki/TiddlyWiki5/pull/9882 +github-contributors: pmario +type: text/vnd.tiddlywiki + +Several wikitext parser rules emitted parse tree nodes with missing or incorrect `start` and `end` source positions. Accurate positions are needed by tooling that maps rendered output back to the source text. + +* ''Inline code'' (`` `code` ``): the text node `end` no longer runs past the closing backtick, so the span covers only the code text. + +* ''Suppressed external links'' (`~https://example.com`): the text node now starts after the suppressing `~`, matching the plain text that is emitted. + +* ''Suppressed wikilinks'' (`~SuppressedLink`): the text node now carries `start` and `end` positions. Previously it had none. + +* ''Import filter'' (`\import`): the filter start position was set to the whole source string instead of the current parse position. It now uses the correct offset.