Fix invalid source positions in parse tree nodes (#9882)

* Fix invalid source positions in parse tree nodes

Several wikitext rules recorded start and end offsets that did not match
the source they represent. The offsets are only used for source position
mapping, not rendering or serialization, so output is unchanged and
source.slice(start, end) now equals the node text.

* import: the filter attribute start was set to the whole source string
  instead of the numeric offset. It used this.parser.source where it
  meant this.parser.pos, so every \import node carried a copy of the
  entire source as its start.

* codeinline: the text node end pointed past the closing backtick, so the
  span included the closing delimiter and could read one or two characters
  too far. It now ends where the content ends.

* wikilinkprefix and extlink: a suppressed link such as ~SomeLink or
  ~http://example.com produced a text node whose span included the
  leading ~ while its text did not. The span now starts after the ~.

* Add 5.4.1 change note for #9882

Document the parse tree source position fixes: inline code, suppressed
external links, suppressed wikilinks, and the import filter rule.

* Add tests for parse tree source positions

Cover the four wikitext parser rules whose `start` and `end` offsets are
corrected in this PR: codeinline, extlink, wikilinkprefix, and import.

core/modules/parsers/wikiparser/rules/codeinline.js

@@ -31,13 +31,16 @@ exports.parse = function() {
 	reEnd.lastIndex = this.parser.pos;
 	var match = reEnd.exec(this.parser.source),
 		text,
-		start = this.parser.pos;
+		start = this.parser.pos,
+		textEnd;
 	// Process the text
 	if(match) {
 		text = this.parser.source.substring(this.parser.pos,match.index);
+		textEnd = match.index;
 		this.parser.pos = match.index + match[0].length;
 	} else {
 		text = this.parser.source.substr(this.parser.pos);
+		textEnd = this.parser.sourceLength;
 		this.parser.pos = this.parser.sourceLength;
 	}
 	return [{
@@ -47,7 +50,7 @@ exports.parse = function() {
 			type: "text",
 			text: text,
 			start: start,
-			end: this.parser.pos
+			end: textEnd
 		}]
 	}];
 };

core/modules/parsers/wikiparser/rules/extlink.js

@@ -32,7 +32,8 @@ exports.parse = function() {
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(this.match[0].substr(0,1) === "~") {
-		return [{type: "text", text: this.match[0].substr(1), start: start, end: this.parser.pos}];
+		// Start after the suppressing "~" so the span matches the plain text
+		return [{type: "text", text: this.match[0].substr(1), start: start + 1, end: this.parser.pos}];
 	} else {
 		return [{
 			type: "element",

core/modules/parsers/wikiparser/rules/import.js

@@ -34,7 +34,7 @@ exports.parse = function() {
 	// Parse the filter terminated by a line break
 	var reMatch = /(.*)(?:$|\r?\n)/mg;
 	reMatch.lastIndex = this.parser.pos;
-	var filterStart = this.parser.source;
+	var filterStart = this.parser.pos;
 	var match = reMatch.exec(this.parser.source);
 	this.parser.pos = reMatch.lastIndex;
 	// Parse tree nodes to return

core/modules/parsers/wikiparser/rules/wikilinkprefix.js

@@ -27,9 +27,11 @@ Parse the most recent match
 */
 exports.parse = function() {
 	// Get the details of the match
-	var linkText = this.match[0];
+	var linkText = this.match[0],
+		// Start after the suppressing "~" so the span matches the plain text
+		start = this.parser.pos + 1;
 	// Move past the wikilink
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Return the link without unwikilink character as plain text
-	return [{type: "text", text: linkText.substr(1)}];
+	return [{type: "text", text: linkText.substr(1), start: start, end: this.parser.pos}];
 };

editions/test/tiddlers/tests/test-parsetree-positions.js

@@ -0,0 +1,60 @@
+/*\
+title: test-parsetree-positions.js
+type: application/javascript
+tags: [[$:/tags/test-spec]]
+
+Regression tests for #9882: wikitext parser rules must emit accurate
+`start`/`end` source positions on their parse tree nodes. Tooling that maps
+rendered output back to the source text relies on these offsets.
+
+\*/
+
+"use strict";
+
+describe("Parse tree source position tests (#9882)", function() {
+
+	// Create a wiki
+	var wiki = $tw.test.wiki();
+
+	// Define a parsing shortcut
+	var parse = function(text) {
+		return wiki.parseText("text/vnd.tiddlywiki",text).tree;
+	};
+
+	it("should give inline code runs a text node that spans only the code, not the backticks", function() {
+		// codeinline.js: `code` gives a text node "code" spanning offsets 1 to 5. The closing backtick at 5 is excluded.
+		// Bug: `end` was set to `this.parser.pos`, which sits past the closing backtick, so end was 6 and the span swallowed the backtick.
+		expect(parse("`code`")).toEqual(
+			[ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 6, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 6, children: [ { type: "text", text: "code", start: 1, end: 5 } ] } ] } ]
+		);
+		// ``a`b`` gives text "a`b" spanning 2 to 5. `end` must be the offset of the closing marker whatever its length.
+		// Bug: `end` was `this.parser.pos` (7), two characters past the code, so it also swallowed the closing ``.
+		expect(parse("``a`b``")).toEqual(
+			[ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 7, children: [ { type: "element", tag: "code", rule: "codeinline", start: 0, end: 7, children: [ { type: "text", text: "a`b", start: 2, end: 5 } ] } ] } ]
+		);
+	});
+
+	it("should start the text node of a suppressed external link after the ~", function() {
+		// extlink.js: ~https://example.com/ emits the plain text "https://example.com/", which spans offsets 1 to 21.
+		// Bug: `start` was the offset of the ~ (0), so the span was one character too wide and began on the ~ that the text omits.
+		expect(parse("~https://example.com/")).toEqual(
+			[ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 21, children: [ { type: "text", text: "https://example.com/", start: 1, end: 21, rule: "extlink" } ] } ]
+		);
+	});
+
+	it("should give a suppressed wikilink's text node source positions", function() {
+		// wikilinkprefix.js: ~SuppressedLink emits the plain text "SuppressedLink", spanning offsets 1 to 15.
+		// Bug: the text node carried no `start`/`end` at all. The parser framework then defaulted `start` to the ~ offset (0).
+		expect(parse("~SuppressedLink")).toEqual(
+			[ { type: "element", tag: "p", rule: "parseblock", start: 0, end: 15, children: [ { type: "text", text: "SuppressedLink", start: 1, end: 15, rule: "wikilinkprefix" } ] } ]
+		);
+	});
+
+	it("should record the filter's start offset for an \\import pragma", function() {
+		// import.js: \import [tag[x]] records the filter value "[tag[x]]" starting at offset 8, right after "\import ".
+		// Bug: `filterStart` was assigned `this.parser.source` (the whole source string) instead of `this.parser.pos`, so `start` was a string, not an offset.
+		expect(parse("\\import [tag[x]]\n")).toEqual(
+			[ { type: "importvariables", rule: "import", start: 0, end: 16, attributes: { filter: { type: "string", value: "[tag[x]]", start: 8, end: 16 } }, children: [] } ]
+		);
+	});
+});

editions/tw5.com/tiddlers/releasenotes/5.4.1/#9882/#9882-fix-invalid-source-positions.tid

@@ -0,0 +1,21 @@
+title: $:/changenotes/5.4.1/#9882
+created: 20260701142414000
+modified: 20260701142414000
+description: Fix invalid source positions in parse tree nodes
+release: 5.4.1
+tags: $:/tags/ChangeNote
+change-type: bugfix
+change-category: internal
+github-links: https://github.com/TiddlyWiki/TiddlyWiki5/pull/9882
+github-contributors: pmario
+type: text/vnd.tiddlywiki
+
+Several wikitext parser rules emitted parse tree nodes with missing or incorrect `start` and `end` source positions. Accurate positions are needed by tooling that maps rendered output back to the source text.
+
+* ''Inline code'' (`` `code` ``): the text node `end` no longer runs past the closing backtick, so the span covers only the code text.
+
+* ''Suppressed external links'' (`~https://example.com`): the text node now starts after the suppressing `~`, matching the plain text that is emitted.
+
+* ''Suppressed wikilinks'' (`~SuppressedLink`): the text node now carries `start` and `end` positions. Previously it had none.
+
+* ''Import filter'' (`\import`): the filter start position was set to the whole source string instead of the current parse position. It now uses the correct offset.