From c308fc44c46bdc3733a5e0dc7344b05935b3dbcc Mon Sep 17 00:00:00 2001 From: Mario Pietsch Date: Wed, 8 Jul 2026 11:03:11 +0200 Subject: [PATCH] Fix invalid source positions in parse tree nodes (#9882) * Fix invalid source positions in parse tree nodes Several wikitext rules recorded start and end offsets that did not match the source they represent. The offsets are only used for source position mapping, not rendering or serialization, so output is unchanged and source.slice(start, end) now equals the node text. * import: the filter attribute start was set to the whole source string instead of the numeric offset. It used this.parser.source where it meant this.parser.pos, so every \import node carried a copy of the entire source as its start. * codeinline: the text node end pointed past the closing backtick, so the span included the closing delimiter and could read one or two characters too far. It now ends where the content ends. * wikilinkprefix and extlink: a suppressed link such as ~SomeLink or ~http://example.com produced a text node whose span included the leading ~ while its text did not. The span now starts after the ~. * Add 5.4.1 change note for #9882 Document the parse tree source position fixes: inline code, suppressed external links, suppressed wikilinks, and the import filter rule. * Add tests for parse tree source positions Cover the four wikitext parser rules whose `start` and `end` offsets are corrected in this PR: codeinline, extlink, wikilinkprefix, and import. --- .../parsers/wikiparser/rules/codeinline.js | 7 ++- .../parsers/wikiparser/rules/extlink.js | 3 +- .../parsers/wikiparser/rules/import.js | 2 +- .../wikiparser/rules/wikilinkprefix.js | 6 +- .../tests/test-parsetree-positions.js | 60 +++++++++++++++++++ .../#9882-fix-invalid-source-positions.tid | 21 +++++++ 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 editions/test/tiddlers/tests/test-parsetree-positions.js create mode 100644 editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid diff --git a/core/modules/parsers/wikiparser/rules/codeinline.js b/core/modules/parsers/wikiparser/rules/codeinline.js index 5606ba478..73e3484aa 100644 --- a/core/modules/parsers/wikiparser/rules/codeinline.js +++ b/core/modules/parsers/wikiparser/rules/codeinline.js @@ -31,13 +31,16 @@ exports.parse = function() { reEnd.lastIndex = this.parser.pos; var match = reEnd.exec(this.parser.source), text, - start = this.parser.pos; + start = this.parser.pos, + textEnd; // Process the text if(match) { text = this.parser.source.substring(this.parser.pos,match.index); + textEnd = match.index; this.parser.pos = match.index + match[0].length; } else { text = this.parser.source.substr(this.parser.pos); + textEnd = this.parser.sourceLength; this.parser.pos = this.parser.sourceLength; } return [{ @@ -47,7 +50,7 @@ exports.parse = function() { type: "text", text: text, start: start, - end: this.parser.pos + end: textEnd }] }]; }; diff --git a/core/modules/parsers/wikiparser/rules/extlink.js b/core/modules/parsers/wikiparser/rules/extlink.js index 7a2485f1c..ca8c13012 100644 --- a/core/modules/parsers/wikiparser/rules/extlink.js +++ b/core/modules/parsers/wikiparser/rules/extlink.js @@ -32,7 +32,8 @@ exports.parse = function() { this.parser.pos = this.matchRegExp.lastIndex; // Create the link unless it is suppressed if(this.match[0].substr(0,1) === "~") { - return [{type: "text", text: this.match[0].substr(1), start: start, end: this.parser.pos}]; + // Start after the suppressing "~" so the span matches the plain text + return [{type: "text", text: this.match[0].substr(1), start: start + 1, end: this.parser.pos}]; } else { return [{ type: "element", diff --git a/core/modules/parsers/wikiparser/rules/import.js b/core/modules/parsers/wikiparser/rules/import.js index 65376f8be..fe291f8e3 100644 --- a/core/modules/parsers/wikiparser/rules/import.js +++ b/core/modules/parsers/wikiparser/rules/import.js @@ -34,7 +34,7 @@ exports.parse = function() { // Parse the filter terminated by a line break var reMatch = /(.*)(?:$|\r?\n)/mg; reMatch.lastIndex = this.parser.pos; - var filterStart = this.parser.source; + var filterStart = this.parser.pos; var match = reMatch.exec(this.parser.source); this.parser.pos = reMatch.lastIndex; // Parse tree nodes to return diff --git a/core/modules/parsers/wikiparser/rules/wikilinkprefix.js b/core/modules/parsers/wikiparser/rules/wikilinkprefix.js index c1c5db1f6..a1f63293e 100644 --- a/core/modules/parsers/wikiparser/rules/wikilinkprefix.js +++ b/core/modules/parsers/wikiparser/rules/wikilinkprefix.js @@ -27,9 +27,11 @@ Parse the most recent match */ exports.parse = function() { // Get the details of the match - var linkText = this.match[0]; + var linkText = this.match[0], + // Start after the suppressing "~" so the span matches the plain text + start = this.parser.pos + 1; // Move past the wikilink this.parser.pos = this.matchRegExp.lastIndex; // Return the link without unwikilink character as plain text - return [{type: "text", text: linkText.substr(1)}]; + return [{type: "text", text: linkText.substr(1), start: start, end: this.parser.pos}]; }; diff --git a/editions/test/tiddlers/tests/test-parsetree-positions.js b/editions/test/tiddlers/tests/test-parsetree-positions.js new file mode 100644 index 000000000..e41dc0648 --- /dev/null +++ b/editions/test/tiddlers/tests/test-parsetree-positions.js @@ -0,0 +1,60 @@ +/*\ +title: test-parsetree-positions.js +type: application/javascript +tags: [[$:/tags/test-spec]] + +Regression tests for #9882: wikitext parser rules must emit accurate +`start`/`end` source positions on their parse tree nodes. Tooling that maps +rendered output back to the source text relies on these offsets. + +\*/ + +"use strict"; + +describe("Parse tree source position tests (#9882)", function() { + + // Create a wiki + var wiki = $tw.test.wiki(); + + // Define a parsing shortcut + var parse = function(text) { + return wiki.parseText("text/vnd.tiddlywiki",text).tree; + }; + + it("should give inline code runs a text node that spans only the code, not the backticks", function() { + // codeinline.js: `code` gives a text node "code" spanning offsets 1 to 5. The closing backtick at 5 is excluded. + // Bug: `end` was set to `this.parser.pos`, which sits past the closing backtick, so end was 6 and the span swallowed the backtick. + expect(parse("`code`")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 6, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 6, children: [ { type: "text", text: "code", start: 1, end: 5 } ] } ] } ] + ); + // ``a`b`` gives text "a`b" spanning 2 to 5. `end` must be the offset of the closing marker whatever its length. + // Bug: `end` was `this.parser.pos` (7), two characters past the code, so it also swallowed the closing ``. + expect(parse("``a`b``")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 7, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 7, children: [ { type: "text", text: "a`b", start: 2, end: 5 } ] } ] } ] + ); + }); + + it("should start the text node of a suppressed external link after the ~", function() { + // extlink.js: ~https://example.com/ emits the plain text "https://example.com/", which spans offsets 1 to 21. + // Bug: `start` was the offset of the ~ (0), so the span was one character too wide and began on the ~ that the text omits. + expect(parse("~https://example.com/")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 21, children: [ { type: "text", text: "https://example.com/", start: 1, end: 21, rule: "extlink" } ] } ] + ); + }); + + it("should give a suppressed wikilink's text node source positions", function() { + // wikilinkprefix.js: ~SuppressedLink emits the plain text "SuppressedLink", spanning offsets 1 to 15. + // Bug: the text node carried no `start`/`end` at all. The parser framework then defaulted `start` to the ~ offset (0). + expect(parse("~SuppressedLink")).toEqual( + [ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 15, children: [ { type: "text", text: "SuppressedLink", start: 1, end: 15, rule: "wikilinkprefix" } ] } ] + ); + }); + + it("should record the filter's start offset for an \\import pragma", function() { + // import.js: \import [tag[x]] records the filter value "[tag[x]]" starting at offset 8, right after "\import ". + // Bug: `filterStart` was assigned `this.parser.source` (the whole source string) instead of `this.parser.pos`, so `start` was a string, not an offset. + expect(parse("\\import [tag[x]]\n")).toEqual( + [ { type: "importvariables", rule: "import", start: 0, end: 16, attributes: { filter: { type: "string", value: "[tag[x]]", start: 8, end: 16 } }, children: [] } ] + ); + }); +}); diff --git a/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid b/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid new file mode 100644 index 000000000..b833ec348 --- /dev/null +++ b/editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid @@ -0,0 +1,21 @@ +title: $:/changenotes/5.4.1/#9882 +created: 20260701142414000 +modified: 20260701142414000 +description: Fix invalid source positions in parse tree nodes +release: 5.4.1 +tags: $:/tags/ChangeNote +change-type: bugfix +change-category: internal +github-links: https://github.com/TiddlyWiki/TiddlyWiki5/pull/9882 +github-contributors: pmario +type: text/vnd.tiddlywiki + +Several wikitext parser rules emitted parse tree nodes with missing or incorrect `start` and `end` source positions. Accurate positions are needed by tooling that maps rendered output back to the source text. + +* ''Inline code'' (`` `code` ``): the text node `end` no longer runs past the closing backtick, so the span covers only the code text. + +* ''Suppressed external links'' (`~https://example.com`): the text node now starts after the suppressing `~`, matching the plain text that is emitted. + +* ''Suppressed wikilinks'' (`~SuppressedLink`): the text node now carries `start` and `end` positions. Previously it had none. + +* ''Import filter'' (`\import`): the filter start position was set to the whole source string instead of the current parse position. It now uses the correct offset.