Basic Backtranscludes filter (#6081)

* feat: add transcludes and backtranscludes filter and its relying indexer

* feat: add test about backtranscludes

* docs: add doc about transcludes and backtranscludes Operator

* refactor: merge backlinks and backtranscludes indexer

* fix: test not executed

* fix: latest transclude use $tiddler instead of tiddler

* feat: A tiddler transclude with template will still use the tiddler as result.

* docs: wrong comment

core/modules/filters/backtranscludes.js

@@ -0,0 +1,26 @@
+/*\
+title: $:/core/modules/filters/backtranscludes.js
+type: application/javascript
+module-type: filteroperator
+
+Filter operator for returning all the backtranscludes from a tiddler
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Export our filter function
+*/
+exports.backtranscludes = function(source,operator,options) {
+	var results = [];
+	source(function(tiddler,title) {
+		$tw.utils.pushTop(results,options.wiki.getTiddlerBacktranscludes(title));
+	});
+	return results;
+};
+
+})();

core/modules/filters/transcludes.js

@@ -0,0 +1,26 @@
+/*\
+title: $:/core/modules/filters/transcludes.js
+type: application/javascript
+module-type: filteroperator
+
+Filter operator for returning all the transcludes from a tiddler
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Export our filter function
+*/
+exports.transcludes = function(source,operator,options) {
+	var results = new $tw.utils.LinkedList();
+	source(function(tiddler,title) {
+		results.pushTop(options.wiki.getTiddlerTranscludes(title));
+	});
+	return results.toArray();
+};
+
+})();

core/modules/indexers/back-indexer.js

@@ -0,0 +1,119 @@
+/*\
+title: $:/core/modules/indexers/back-indexer.js
+type: application/javascript
+module-type: indexer
+
+By parsing the tiddler text, indexes the tiddlers' back links, back transclusions, block level back links.
+
+\*/
+function BackIndexer(wiki) {
+	this.wiki = wiki;
+}
+
+BackIndexer.prototype.init = function() {
+	this.subIndexers = {
+		link: new BackSubIndexer(this,"extractLinks"),
+		transclude: new BackSubIndexer(this,"extractTranscludes"),
+	};
+};
+
+BackIndexer.prototype.rebuild = function() {
+	$tw.utils.each(this.subIndexers,function(subIndexer) {
+		subIndexer.rebuild();
+	});
+};
+
+BackIndexer.prototype.update = function(updateDescriptor) {
+	$tw.utils.each(this.subIndexers,function(subIndexer) {
+		subIndexer.update(updateDescriptor);
+	});
+};
+function BackSubIndexer(indexer,extractor) {
+	this.wiki = indexer.wiki;
+	this.indexer = indexer;
+	this.extractor = extractor;
+	/**
+	 * {
+	 *   [target title, e.g. tiddler title being linked to]:
+	 *     {
+	 * 		   [source title, e.g. tiddler title that has link syntax in its text]: true
+	 * 	   }
+	 * }
+	 */
+	this.index = null;
+}
+
+BackSubIndexer.prototype.init = function() {
+	// lazy init until first lookup
+	this.index = null;
+}
+
+BackSubIndexer.prototype._init = function() {
+	this.index = Object.create(null);
+	var self = this;
+	this.wiki.forEachTiddler(function(sourceTitle,tiddler) {
+		var newTargets = self._getTarget(tiddler);
+		$tw.utils.each(newTargets, function(target) {
+			if(!self.index[target]) {
+				self.index[target] = Object.create(null);
+			}
+			self.index[target][sourceTitle] = true;
+		});
+	});
+}
+
+BackSubIndexer.prototype.rebuild = function() {
+	this.index = null;
+}
+
+/*
+* Get things that is being referenced in the text, e.g. tiddler names in the link syntax.
+*/
+BackSubIndexer.prototype._getTarget = function(tiddler) {
+	var parser = this.wiki.parseText(tiddler.fields.type, tiddler.fields.text, {});
+	if(parser) {
+		return this.wiki[this.extractor](parser.tree);
+	}
+	return [];
+}
+
+BackSubIndexer.prototype.update = function(updateDescriptor) {
+	// lazy init/update until first lookup
+	if(!this.index) {
+		return;
+	}
+	var newTargets = [],
+	    oldTargets = [],
+	    self = this;
+	if(updateDescriptor.old.exists) {
+		oldTargets = this._getTarget(updateDescriptor.old.tiddler);
+	}
+	if(updateDescriptor.new.exists) {
+		newTargets = this._getTarget(updateDescriptor.new.tiddler);
+	}
+
+	$tw.utils.each(oldTargets,function(target) {
+		if(self.index[target]) {
+			delete self.index[target][updateDescriptor.old.tiddler.fields.title];
+		}
+	});
+	$tw.utils.each(newTargets,function(target) {
+		if(!self.index[target]) {
+			self.index[target] = Object.create(null);
+		}
+		self.index[target][updateDescriptor.new.tiddler.fields.title] = true;
+	});
+}
+
+BackSubIndexer.prototype.lookup = function(title) {
+	if(!this.index) {
+		this._init();
+	}
+	if(this.index[title]) {
+		return Object.keys(this.index[title]);
+	} else {
+		return [];
+	}
+}
+
+exports.BackIndexer = BackIndexer;

core/modules/indexers/backlinks-index.js

@@ -1,86 +0,0 @@
-/*\
-title: $:/core/modules/indexers/backlinks-indexer.js
-type: application/javascript
-module-type: indexer
-
-Indexes the tiddlers' backlinks
-
-\*/
-(function(){
-
-/*jslint node: true, browser: true */
-/*global modules: false */
-"use strict";
-
-
-function BacklinksIndexer(wiki) {
-	this.wiki = wiki;
-}
-
-BacklinksIndexer.prototype.init = function() {
-	this.index = null;
-}
-
-BacklinksIndexer.prototype.rebuild = function() {
-	this.index = null;
-}
-
-BacklinksIndexer.prototype._getLinks = function(tiddler) {
-	var parser =  this.wiki.parseText(tiddler.fields.type, tiddler.fields.text, {});
-	if(parser) {
-		return this.wiki.extractLinks(parser.tree);
-	}
-	return [];
-}
-
-BacklinksIndexer.prototype.update = function(updateDescriptor) {
-	if(!this.index) {
-		return;
-	}
-	var newLinks = [],
-	    oldLinks = [],
-	    self = this;
-	if(updateDescriptor.old.exists) {
-		oldLinks = this._getLinks(updateDescriptor.old.tiddler);
-	}
-	if(updateDescriptor.new.exists) {
-		newLinks = this._getLinks(updateDescriptor.new.tiddler);
-	}
-
-	$tw.utils.each(oldLinks,function(link) {
-		if(self.index[link]) {
-			delete self.index[link][updateDescriptor.old.tiddler.fields.title];
-		}
-	});
-	$tw.utils.each(newLinks,function(link) {
-		if(!self.index[link]) {
-			self.index[link] = Object.create(null);
-		}
-		self.index[link][updateDescriptor.new.tiddler.fields.title] = true;
-	});
-}
-
-BacklinksIndexer.prototype.lookup = function(title) {
-	if(!this.index) {
-		this.index = Object.create(null);
-		var self = this;
-		this.wiki.forEachTiddler(function(title,tiddler) {
-			var links = self._getLinks(tiddler);
-			$tw.utils.each(links, function(link) {
-				if(!self.index[link]) {
-					self.index[link] = Object.create(null);
-				}
-				self.index[link][title] = true;
-			});
-		});
-	}
-	if(this.index[title]) {
-		return Object.keys(this.index[title]);
-	} else {
-		return [];
-	}
-}
-
-exports.BacklinksIndexer = BacklinksIndexer;
-
-})();

core/modules/wiki.js

@@ -534,8 +534,8 @@ Return an array of tiddler titles that link to the specified tiddler
 */
 exports.getTiddlerBacklinks = function(targetTitle) {
 	var self = this,
-		backlinksIndexer = this.getIndexer("BacklinksIndexer"),
-		backlinks = backlinksIndexer && backlinksIndexer.lookup(targetTitle);
+		backIndexer = this.getIndexer("BackIndexer"),
+		backlinks = backIndexer && backIndexer.subIndexers.link.lookup(targetTitle);
 
 	if(!backlinks) {
 		backlinks = [];
@@ -549,6 +549,68 @@ exports.getTiddlerBacklinks = function(targetTitle) {
 	return backlinks;
 };
 
+
+/*
+Return an array of tiddler titles that are directly transcluded within the given parse tree
+ */
+exports.extractTranscludes = function(parseTreeRoot) {
+	// Count up the transcludes
+	var transcludes = [],
+		checkParseTree = function(parseTree, parentNode) {
+			for(var t=0; t<parseTree.length; t++) {
+				var parseTreeNode = parseTree[t];
+				if(parseTreeNode.type === "transclude" && parseTreeNode.attributes.$tiddler && parseTreeNode.attributes.$tiddler.type === "string") {
+					var value;
+					// if it is Transclusion with Templates like `{{Index||$:/core/ui/TagTemplate}}`, the `$tiddler` will point to the template. We need to find the actual target tiddler from parent node
+					if(parentNode && parentNode.type === "tiddler" && parentNode.attributes.tiddler && parentNode.attributes.tiddler.type === "string") {
+						value = parentNode.attributes.tiddler.value;
+					} else {
+						value = parseTreeNode.attributes.$tiddler.value;
+					}
+					if(transcludes.indexOf(value) === -1) {
+						transcludes.push(value);
+					}
+				}
+				if(parseTreeNode.children) {
+					checkParseTree(parseTreeNode.children, parseTreeNode);
+				}
+			}
+		};
+	checkParseTree(parseTreeRoot);
+	return transcludes;
+};
+
+
+/*
+Return an array of tiddler titles that are transcluded from the specified tiddler
+*/
+exports.getTiddlerTranscludes = function(title) {
+	var self = this;
+	// We'll cache the transcludes so they only get computed if the tiddler changes
+	return this.getCacheForTiddler(title,"transcludes",function() {
+		// Parse the tiddler
+		var parser = self.parseTiddler(title);
+		if(parser) {
+			return self.extractTranscludes(parser.tree);
+		}
+		return [];
+	});
+};
+
+/*
+Return an array of tiddler titles that transclude to the specified tiddler
+*/
+exports.getTiddlerBacktranscludes = function(targetTitle) {
+	var self = this,
+		backIndexer = this.getIndexer("BackIndexer"),
+		backtranscludes = backIndexer && backIndexer.subIndexers.transclude.lookup(targetTitle);
+
+	if(!backtranscludes) {
+		backtranscludes = [];
+	}
+	return backtranscludes;
+};
+
 /*
 Return a hashmap of tiddler titles that are referenced but not defined. Each value is the number of times the missing tiddler is referenced
 */

editions/test/tiddlers/tests/test-backtranscludes.js

@@ -0,0 +1,148 @@
+/*\
+title: test-backtranscludes.js
+type: application/javascript
+tags: $:/tags/test-spec
+
+Tests the backtranscludes mechanism.
+
+\*/
+(function(){
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+describe('Backtranscludes tests', function() {
+	describe('a tiddler with no transcludes to it', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: ''});
+
+		it('should have no backtranscludes', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('');
+		});
+	});
+
+	describe('A tiddler added to the wiki with a transclude to it', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: 'something'});
+
+		wiki.addTiddler({
+			title: 'TestOutgoing',
+			text: 'A transclude to {{TestIncoming}}'});
+
+		it('should have a backtransclude', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+		});
+	});
+
+	describe('A tiddler transclude with template will still use the tiddler as result.', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: 'something'});
+
+		wiki.addTiddler({
+			title: 'TestOutgoing',
+			text: 'A transclude to {{TestIncoming||$:/core/ui/TagTemplate}}'});
+
+		it('should have a backtransclude', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+		});
+	});
+
+	describe('A tiddler that has a transclude added to it later', function() {
+		it('should have an additional backtransclude', function() {
+			var wiki = new $tw.Wiki();
+
+			wiki.addTiddler({
+				title: 'TestIncoming',
+				text: ''});
+
+			wiki.addTiddler({
+				title: 'TestOutgoing',
+				text: 'A transclude to {{TestIncoming}}'});
+
+			wiki.addTiddler({
+				title: 'TestOutgoing2',
+				text: 'Nothing yet!'});
+
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+
+			wiki.addTiddler({
+				title: 'TestOutgoing2',
+				text: 'Updated with transclude to {{TestIncoming}}'});
+
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing,TestOutgoing2');
+		});
+	});
+
+	describe('A tiddler that has a transclude remove from it later', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: ''});
+
+		wiki.addTiddler({
+			title: 'TestOutgoing',
+			text: 'A transclude to {{TestIncoming}}'});
+
+		it('should have one fewer backtransclude', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+
+			wiki.addTiddler({
+				title: 'TestOutgoing',
+				text: 'No transclude to ~TestIncoming'});
+
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('');
+		});
+	});
+
+	describe('A tiddler transcludeing to another that gets renamed', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: ''});
+
+		wiki.addTiddler({
+			title: 'TestOutgoing',
+			text: 'A transclude to {{TestIncoming}}'});
+
+		it('should have its name changed in the backtranscludes', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+
+			wiki.renameTiddler('TestOutgoing', 'TestExtroverted');
+
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestExtroverted');
+		});
+	});
+
+	describe('A tiddler transcludeing to another that gets deleted', function() {
+		var wiki = new $tw.Wiki();
+
+		wiki.addTiddler({
+			title: 'TestIncoming',
+			text: ''});
+
+		wiki.addTiddler({
+			title: 'TestOutgoing',
+			text: 'A transclude to {{TestIncoming}}'});
+
+		it('should be removed from backtranscludes', function() {
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('TestOutgoing');
+
+			wiki.deleteTiddler('TestOutgoing');
+
+			expect(wiki.filterTiddlers('TestIncoming +[backtranscludes[]]').join(',')).toBe('');
+		});
+	});
+});
+
+})();

editions/tw5.com/tiddlers/filters/backtranscludes.tid

@@ -0,0 +1,13 @@
+created: 20211002204500000
+tags: [[Filter Operators]]
+title: backtranscludes Operator
+type: text/vnd.tiddlywiki
+caption: backtranscludes
+op-purpose: find the titles that transcludes to each input title
+op-input: a [[selection of titles|Title Selection]]
+op-parameter: none
+op-output: any non-[[system|SystemTiddlers]] titles that contain [[transclusion|Transclusion]] to the input titles
+
+Each input title is processed in turn. The corresponding tiddler's list of backtranscludes is generated, sorted alphabetically by title, and then [[dominantly appended|Dominant Append]] to the operator's overall output.
+
+<<.operator-examples "backtranscludes">>

editions/tw5.com/tiddlers/filters/examples/backtransclude.tid

@@ -0,0 +1,7 @@
+tags: [[backtranscludes Operator]] [[Operator Examples]]
+title: backtranscludes Operator (Examples)
+type: text/vnd.tiddlywiki
+
+<<.operator-example 1 "[[Motovun Jack.jpg]backtranscludes[]]">>
+
+<<.operator-example 2 "[[Transclusion]backtranscludes[]]">>

editions/tw5.com/tiddlers/filters/examples/transclude.tid

@@ -0,0 +1,5 @@
+tags: [[transcludes Operator]] [[Operator Examples]]
+title: transcludes Operator (Examples)
+type: text/vnd.tiddlywiki
+
+<<.operator-example 1 "[[Images in WikiText]transcludes[]]">>

editions/tw5.com/tiddlers/filters/transcludes.tid

@@ -0,0 +1,13 @@
+created: 20211002204500000
+tags: [[Filter Operators]] [[Common Operators]]
+title: transcludes Operator
+type: text/vnd.tiddlywiki
+caption: transcludes
+op-purpose: find the titles linked to by each input title
+op-input: a [[selection of titles|Title Selection]]
+op-parameter: none
+op-output: the titles to which the input tiddlers [[transcludes|Transclusion]]
+
+Each input title is processed in turn. The corresponding tiddler's list of transcludes is generated, in the order in which they appear in the tiddler's text, and [[dominantly appended|Dominant Append]] to the operator's overall output.
+
+<<.operator-examples "transcludes">>