Add :sort filter run prefix (#5653)

* Add :sort filter run prefix, docs and tests. Also extended .utils.makeCompareFunction with a flag for caseSensitivity.

* Documentation updates

* Move case sensitivity handling entirely to utils method so it is reusable

core/modules/filterrunprefixes/sort.js

@@ -0,0 +1,58 @@
+/*\
+title: $:/core/modules/filterrunprefixes/sort.js
+type: application/javascript
+module-type: filterrunprefix
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+/*
+Export our filter prefix function
+*/
+exports.sort = function(operationSubFunction,options) {
+	return function(results,source,widget) {
+		if(results.length > 0) {
+			var suffixes = options.suffixes,
+				sortType = (suffixes[0] && suffixes[0][0]) ? suffixes[0][0] : "string",
+				invert = suffixes[1] ? (suffixes[1].indexOf("reverse") !== -1) : false,
+				isCaseSensitive = suffixes[1] ? (suffixes[1].indexOf("casesensitive") !== -1) : false,
+				inputTitles = results.toArray(),
+				sortKeys = [],
+				indexes = new Array(inputTitles.length),
+				compareFn;	
+			results.each(function(title) {
+				var key = operationSubFunction(options.wiki.makeTiddlerIterator([title]),{
+						getVariable: function(name) {
+							switch(name) {
+								case "currentTiddler":
+									return "" + title;
+								default:
+									return widget.getVariable(name);
+							}
+						}
+					});
+				sortKeys.push(key[0] || "");
+			});
+			results.clear();
+			// Prepare an array of indexes to sort
+			for(var t=0; t<inputTitles.length; t++) {
+				indexes[t] = t;
+			}
+			// Sort the indexes
+			compareFn = $tw.utils.makeCompareFunction(sortType,{defaultType: "string", invert:invert, isCaseSensitive:isCaseSensitive});
+			indexes = indexes.sort(function(a,b) {
+					return compareFn(sortKeys[a],sortKeys[b]);
+			});
+			// Add to results in correct order
+			$tw.utils.each(indexes,function(index) {
+				results.push(inputTitles[index]);
+			});
+		}
+	}
+};
+
+})();

core/modules/utils/utils.js

@@ -876,7 +876,9 @@ exports.stringifyNumber = function(num) {
 
 exports.makeCompareFunction = function(type,options) {
 	options = options || {};
-	var gt = options.invert ? -1 : +1,
+	// set isCaseSensitive to true if not defined in options
+	var isCaseSensitive = (options.isCaseSensitive === false) ? false : true,
+		gt = options.invert ? -1 : +1,
 		lt = options.invert ? +1 : -1,
 		compare = function(a,b) {
 			if(a > b) {
@@ -895,7 +897,11 @@ exports.makeCompareFunction = function(type,options) {
 				return compare($tw.utils.parseInt(a),$tw.utils.parseInt(b));
 			},
 			"string": function(a,b) {
-				return compare("" + a,"" +b);
+				if(!isCaseSensitive) {
+					a = a.toLowerCase();
+					b = b.toLowerCase();
+				}
+				return compare("" + a,"" + b);
 			},
 			"date": function(a,b) {
 				var dateA = $tw.utils.parseDate(a),
@@ -910,6 +916,13 @@ exports.makeCompareFunction = function(type,options) {
 			},
 			"version": function(a,b) {
 				return $tw.utils.compareVersions(a,b);
+			},
+			"alphanumeric": function(a,b) {
+				if(!isCaseSensitive) {
+					a = a.toLowerCase();
+					b = b.toLowerCase();
+				}
+				return options.invert ? b.localeCompare(a,undefined,{numeric: true,sensitivity: "base"}) : a.localeCompare(b,undefined,{numeric: true,sensitivity: "base"});
 			}
 		};
 	return (types[type] || types[options.defaultType] || types.number);

editions/test/tiddlers/tests/test-filters.js

@@ -750,6 +750,7 @@ function runTests(wiki) {
 		rootWidget.setVariable("sort1","[length[]]");
 		rootWidget.setVariable("sort2","[get[text]else[]length[]]");
 		rootWidget.setVariable("sort3","[{!!value}divide{!!cost}]");
+		rootWidget.setVariable("sort4","[{!!title}]");
 		expect(wiki.filterTiddlers("[sortsub:number<sort1>]",anchorWidget).join(",")).toBe("one,hasList,TiddlerOne,has filter,$:/TiddlerTwo,Tiddler Three,$:/ShadowPlugin,a fourth tiddler,filter regexp test");
 		expect(wiki.filterTiddlers("[!sortsub:number<sort1>]",anchorWidget).join(",")).toBe("filter regexp test,a fourth tiddler,$:/ShadowPlugin,$:/TiddlerTwo,Tiddler Three,TiddlerOne,has filter,hasList,one");
 		expect(wiki.filterTiddlers("[sortsub:string<sort1>]",anchorWidget).join(",")).toBe("TiddlerOne,has filter,$:/TiddlerTwo,Tiddler Three,$:/ShadowPlugin,a fourth tiddler,filter regexp test,one,hasList");
@@ -759,6 +760,7 @@ function runTests(wiki) {
 		expect(wiki.filterTiddlers("[sortsub:string<sort2>]",anchorWidget).join(",")).toBe("one,TiddlerOne,hasList,has filter,$:/ShadowPlugin,a fourth tiddler,Tiddler Three,$:/TiddlerTwo,filter regexp test");
 		expect(wiki.filterTiddlers("[!sortsub:string<sort2>]",anchorWidget).join(",")).toBe("filter regexp test,$:/TiddlerTwo,Tiddler Three,a fourth tiddler,$:/ShadowPlugin,has filter,hasList,TiddlerOne,one");
 		expect(wiki.filterTiddlers("[[TiddlerOne]] [[$:/TiddlerTwo]] [[Tiddler Three]] [[a fourth tiddler]] +[!sortsub:number<sort3>]",anchorWidget).join(",")).toBe("$:/TiddlerTwo,Tiddler Three,TiddlerOne,a fourth tiddler");
+		expect(wiki.filterTiddlers("a1 a10 a2 a3 b10 b3 b1 c9 c11 c1 +[sortsub:alphanumeric<sort4>]",anchorWidget).join(",")).toBe("a1,a2,a3,a10,b1,b3,b10,c1,c9,c11");
 	});
 	
 	it("should handle the toggle operator", function() {

editions/test/tiddlers/tests/test-prefixes-filter.js

@@ -253,7 +253,7 @@ describe("'reduce' and 'intersection' filter prefix tests", function() {
 	wiki.addTiddler({
 		title: "Red wine",
 		tags: ["drinks", "wine", "textexample"],
-		text: "This is some more text"
+		text: "This is some more text!"
 	});
 	wiki.addTiddler({
 		title: "Cheesecake",
@@ -265,6 +265,26 @@ describe("'reduce' and 'intersection' filter prefix tests", function() {
 		tags: ["cakes", "food", "textexample"],
 		text: "This is even more text"
 	});
+	wiki.addTiddler({
+		title: "Persian love cake",
+		tags: ["cakes"],
+		text: "An amazing cake worth the effort to make"
+	});
+	wiki.addTiddler({
+		title: "cheesecake",
+		tags: ["cakes"],
+		text: "Everyone likes cheescake"
+	});
+	wiki.addTiddler({
+		title: "chocolate cake",
+		tags: ["cakes"],
+		text: "lower case chocolate cake"
+	});
+	wiki.addTiddler({
+		title: "Pound cake",
+		tags: ["cakes","with tea"],
+		text: "Does anyone eat pound cake?"
+	});
 
 	it("should handle the :reduce filter prefix", function() {
 		expect(wiki.filterTiddlers("[tag[shopping]] :reduce[get[quantity]add<accumulator>]").join(",")).toBe("22");
@@ -341,8 +361,19 @@ describe("'reduce' and 'intersection' filter prefix tests", function() {
 		expect(wiki.filterTiddlers("[tag[textexample]]",anchorWidget).join(",")).toBe("Sparkling water,Red wine,Cheesecake,Chocolate Cake");
 		expect(wiki.filterTiddlers("[tag[textexample]filter<larger-than-18>]",anchorWidget).join(",")).toBe("Red wine,Cheesecake,Chocolate Cake");
 		expect(wiki.filterTiddlers("[tag[textexample]filter<larger-than-18-with-var>]",anchorWidget).join(",")).toBe("Red wine,Cheesecake,Chocolate Cake");
-	})
+	});
 
+	it("should handle the :sort prefix", function() {
+		expect(wiki.filterTiddlers("a1 a10 a2 a3 b10 b3 b1 c9 c11 c1 :sort:alphanumeric[{!!title}]").join(",")).toBe("a1,a2,a3,a10,b1,b3,b10,c1,c9,c11");
+		expect(wiki.filterTiddlers("a1 a10 a2 a3 b10 b3 b1 c9 c11 c1 :sort:alphanumeric:reverse[{!!title}]").join(",")).toBe("c11,c9,c1,b10,b3,b1,a10,a3,a2,a1");
+		expect(wiki.filterTiddlers("[tag[shopping]] :sort:number:[get[price]]").join(",")).toBe("Milk,Chick Peas,Rice Pudding,Brownies");
+		expect(wiki.filterTiddlers("[tag[textexample]] :sort:number:[get[text]length[]]").join(",")).toBe("Sparkling water,Chocolate Cake,Red wine,Cheesecake");
+		expect(wiki.filterTiddlers("[tag[textexample]] :sort:number:reverse[get[text]length[]]").join(",")).toBe("Cheesecake,Red wine,Chocolate Cake,Sparkling water");
+		expect(wiki.filterTiddlers("[tag[notatag]] :sort:number[get[price]]").join(",")).toBe("");
+		expect(wiki.filterTiddlers("[tag[cakes]] :sort:string[{!!title}]").join(",")).toBe("Cheesecake,cheesecake,Chocolate Cake,chocolate cake,Persian love cake,Pound cake");
+		expect(wiki.filterTiddlers("[tag[cakes]] :sort:string:casesensitive[{!!title}]").join(",")).toBe("Cheesecake,Chocolate Cake,Persian love cake,Pound cake,cheesecake,chocolate cake");
+		expect(wiki.filterTiddlers("[tag[cakes]] :sort:string:casesensitive,reverse[{!!title}]").join(",")).toBe("chocolate cake,cheesecake,Pound cake,Persian love cake,Chocolate Cake,Cheesecake");
+	});
 });
 
 })();

editions/tw5.com/tiddlers/filters/sortsub Operator.tid

@@ -1,17 +1,17 @@
+caption: sortsub
 created: 20200424160155182
-modified: 20200424160155182
+modified: 20210428152533501
+op-input: a [[selection of titles|Title Selection]]
+op-neg-output: the input, sorted into reverse order by the result of evaluating subfilter <<.param S>>
+op-output: the input, sorted into ascending order by the result of evaluating subfilter <<.param S>>
+op-parameter: a subfilter to be evaluated
+op-parameter-name: S
+op-purpose: sort the input by the result of evaluating a subfilter for each item
+op-suffix: the type used for the comparison (string, number, integer, date, version), defaulting to string
+op-suffix-name: T
 tags: [[Filter Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]
 title: sortsub Operator
 type: text/vnd.tiddlywiki
-caption: sortsub
-op-purpose: sort the input by the result of evaluating a subfilter for each item
-op-input: a [[selection of titles|Title Selection]]
-op-parameter: a subfilter to be evaluated
-op-parameter-name: S
-op-suffix: the type used for the comparison (string, number, integer, date, version), defaulting to string
-op-suffix-name: T
-op-output: the input, sorted into ascending order by the result of evaluating subfilter <<.param S>>
-op-neg-output: the input, sorted into reverse order by the result of evaluating subfilter <<.param S>>
 
 Each item in the list of input titles is passed to the subfilter in turn. The subfilter transforms the input titles into the form needed for sorting. For example, the subfilter `[length[]]` transforms each input title in the number representing its length, and thus sorts the input titles according to their length.
 
@@ -24,6 +24,7 @@ The suffix <<.place T>> determines how the items are compared and can be:
 * "integer" - invalid integers are interpreted as zero
 * "date" - invalid dates are interpreted as 1st January 1970
 * "version" - invalid versions are interpreted as "v0.0.0"
+* "alphanumeric" - treat items as alphanumerics <<.from-version "5.1.24">>
 
 Note that subfilters should return the same number of items that they are passed. Any missing entries will be treated as zero or the empty string. In particular, when retrieving the value of a field with the [[get Operator]] it is helpful to guard against a missing field value using the [[else Operator]]. For example `[get[myfield]else[default-value]...`.
 

editions/tw5.com/tiddlers/filters/syntax/Filter Expression.tid

@@ -1,5 +1,5 @@
 created: 20150124182421000
-modified: 20201214053032397
+modified: 20210428084144231
 tags: [[Filter Syntax]]
 title: Filter Expression
 type: text/vnd.tiddlywiki
@@ -26,6 +26,8 @@ If a run has:
 * named prefix `:intersection` replaces all filter output so far with titles that are present in the output of this run, as well as the output from previous runs. Forms the input for the next run. <<.from-version "5.1.23">>
 * named prefix `:reduce` replaces all filter output so far with a single item by repeatedly applying a formula to each input title. A typical use is to add up the values in a given field of each input title. <<.from-version "5.1.23">>
 ** [[Examples|Filter Run Prefix (Examples)]]
+* named prefix `:sort` sorts all filter output so far by applying this run to each input title and sorting according to that output. <<.from-version "5.1.24">>
+** See [[Sort Filter Run Prefix]].
 
 <<.tip "Compare named filter run prefix `:filter` with [[filter Operator]] which applies a subfilter to every input title, removing the titles that return an empty result from the subfilter">>
 
@@ -47,7 +49,7 @@ The input of a run is normally a list of all the non-[[shadow|ShadowTiddlers]] t
 
 |Prefix|Input|h
 |`-`, `~`, `=`, `:intersection` or none| <$link to="all Operator">`[all[]]`</$link> tiddler titles, unless otherwise determined by the first [[filter operator|Filter Operators]]|
-|`+`, `:filter`, `:reduce`|the filter output of all previous runs so far|
+|`+`, `:filter`, `:reduce`,`:sort`|the filter output of all previous runs so far|
 
 Precisely because of varying inputs, be aware that both prefixes `-` and `+` do not behave inverse to one another!
 

editions/tw5.com/tiddlers/filters/syntax/Filter Run Prefix (Examples).tid

@@ -1,6 +1,6 @@
 created: 20201117073343969
-modified: 20201208185546667
-tags: [[Filter Syntax]]
+modified: 20210428084013109
+tags: [[Filter Syntax]] [[Filter Run Prefix Examples]]
 title: Filter Run Prefix (Examples)
 type: text/vnd.tiddlywiki
 
@@ -44,3 +44,7 @@ Specifying a default value when input is empty:
 `[tag[non-existent]] :reduce[get[price]multiply{!!quantity}add<accumulator>] :else[[0]]`
 
 <$macrocall $name=".tip" _="""Unlike the [[reduce Operator]], the `:reduce` prefix cannot specify an initial value for the accumulator, so its initial value will always be empty (which is treated as 0 by mathematical operators). So `=1 =2 =3 :reduce[multiply<accumulator>]` will produce 0, not 6. If you need to specify an initial accumulator value, use the [[reduce Operator]]."""/>
+
+!! `:sort` examples
+
+See [[Sort Filter Run Prefix (Examples)]]

editions/tw5.com/tiddlers/filters/syntax/Sort Filter Run Prefix (Examples).tid

@@ -0,0 +1,33 @@
+created: 20210428074912172
+modified: 20210428085746041
+tags: [[Filter Syntax]] [[Sort Filter Run Prefix]] [[Filter Run Prefix Examples]]
+title: Sort Filter Run Prefix (Examples)
+type: text/vnd.tiddlywiki
+
+Sort by title length:
+
+<<.operator-example 1 "[all[tiddlers]] :sort:number[length[]] +[limit[10]]">>
+
+Sort by title length reversed:
+
+<<.operator-example 2 "[all[tiddlers]] :sort:number:reverse[length[]] +[limit[10]]">>
+
+Sort by text length:
+
+<<.operator-example 3 "[all[tiddlers]] :sort:number[get[text]length[]] +[limit[10]]">>
+
+Sort by newest of modified dates:
+
+<<.operator-example 4 "[tag[Field Operators]] :sort:date[get[modified]else[19700101]] +[limit[10]]">>
+
+Sort by title:
+<<.operator-example 5 "[tag[Field Operators]] :sort:string:casesensitive[get[caption]] +[limit[10]]">>
+
+Sort by title in reverse order:
+<<.operator-example 6 "[tag[Field Operators]] :sort:string:casesensitive,reverse[get[caption]] +[limit[10]]">>
+
+Sort as text with case sensitivity:
+<<.operator-example 7 "Apple Banana Orange Grapefruit guava DragonFruit Kiwi apple orange  :sort:string:casesensitive[{!!title}]">>
+
+Sort as text ignoring case:
+<<.operator-example 8 "Apple Banana Orange Grapefruit guava DragonFruit Kiwi apple orange :sort:string:caseinsensitive[{!!title}]">>

editions/tw5.com/tiddlers/filters/syntax/Sort Filter Run Prefix.tid

@@ -0,0 +1,32 @@
+created: 20210428083929749
+modified: 20210428140713422
+tags: [[Filter Syntax]] [[Filter Run Prefix]]
+title: Sort Filter Run Prefix
+type: text/vnd.tiddlywiki
+
+<<.from-version "5.1.24">>
+
+|''purpose'' |sort the input titles by the result of evaluating this filter run for each item |
+|''input'' |all titles from previous filter runs |
+|''suffix'' |the `:sort` filter run prefix uses a rich suffix, see below for details |
+|''output''|the sorted result of previous filter runs |
+
+Each input title from previous runs is passed to this run in turn. The filter run transforms the input titles into the form needed for sorting. For example, the filter run `[length[]]` transforms each input title in to the number representing its length, and thus sorts the input titles according to their length.
+
+Note that within the filter run, the "currentTiddler" variable is set to the title of the tiddler being processed. This permits filter runs like `:sort:number[{!!value}divide{!!cost}]` to be used for computation.
+
+The `:sort` filter run prefix uses an extended syntax that allows for multiple suffixes, some of which are required:
+
+```
+:sort:<type>:<flaglist>[...filter run...]
+
+```
+
+* ''type'': Required. Determines how the items are compared and can be any of: ''string'', ''alphanumeric'', ''number'', ''integer'', ''version'' or ''date''.
+* ''flaglist'': comma separated list of the following flags:
+** ''casesensitive'' or ''caseinsensitive'' (required for types `string` and `alphanumeric`).
+** ''reverse'' to invert the order of the filter run (optional).
+
+Note that filter runs used with the `:sort` prefix should return the same number of items that they are passed. Any missing entries will be treated as zero or the empty string. In particular, when retrieving the value of a field with the [[get Operator]] it is helpful to guard against a missing field value using the [[else Operator]]. For example `[get[myfield]else[default-value]...`.
+
+[[Examples|Sort Filter Run Prefix (Examples)]]