Adds a deserialize filter operator (#7511)

* feat: added deserialize operator, tests and documentation

* fix: correct typo in lingo file

* fix: remove test that fails on node but succeeds in browser due to different availability of DOM deserializer

core/language/en-GB/Misc.multids

@@ -25,6 +25,8 @@ Encryption/RepeatPassword: Repeat password
 Encryption/PasswordNoMatch: Passwords do not match
 Encryption/SetPassword: Set password
 Error/Caption: Error
+Error/DeserializeOperator/MissingOperand: Filter Error: Missing operand for 'deserialize' operator
+Error/DeserializeOperator/UnknownDeserializer: Filter Error: Unknown deserializer provided as operand for the 'deserialize' operator
 Error/Filter: Filter error
 Error/FilterSyntax: Syntax error in filter expression
 Error/FilterRunPrefix: Filter Error: Unknown prefix for filter run

core/modules/filters/deserialize.js

@@ -0,0 +1,39 @@
+/*\
+title: $:/core/modules/filters/deserialize.js
+type: application/javascript
+module-type: filteroperator
+Filter operator for deserializing string data into JSON representing tiddlers
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports["deserialize"] = function(source,operator,options) {
+	var results = [],
+		deserializer;
+	if(operator.operand) {
+		// Get the deserializer identified by the operand
+		deserializer = $tw.Wiki.tiddlerDeserializerModules[operator.operand];
+		if(deserializer) {
+			source(function(tiddler,title) {
+				var tiddlers;
+				try {
+					tiddlers = deserializer(title);
+				} catch(e) {
+					// Return an empty array if we could not extract any tiddlers
+					tiddlers = [];
+				}
+				results.push(JSON.stringify(tiddlers));
+			});
+		} else {
+			return [$tw.language.getString("Error/DeserializeOperator/UnknownDeserializer")];
+		}
+	} else {
+		return [$tw.language.getString("Error/DeserializeOperator/MissingOperand")];
+	}
+	return results;
+}
+
+})();

editions/test/tiddlers/tests/data/deserializers/case 6.tid

@@ -0,0 +1,8 @@
+title: dezerializer test data case 6
+type: application/json
+
+[
+	{"created":"20230601125557184","text":"Before you start storing important information in ~TiddlyWiki it is vital to make sure that you can reliably save changes. See https://tiddlywiki.com/#GettingStarted for details\n\n","title":"GettingStarted","modified":"20230601125601619"},
+    {"created":"20230601125507054","text":"Welcome to \"TiddlyWiki\".\n\nThis is a test tiddler.","tags":"","title":"Hello There \"Welcome\"","modified":"20230601125551144"},
+    {"title":"TiddlyWiki","created":"20130822170700000","modified":"20170127221451610","tags":"Concepts","type":"text/vnd.tiddlywiki","text":"~TiddlyWiki is a rich, interactive tool for manipulating complex data with structure that doesn't easily fit into conventional tools like spreadsheets or wordprocessors.\n\n~TiddlyWiki is designed to fit around your brain, helping you deal with the things that won't fit."}
+]

editions/test/tiddlers/tests/test-deserialize-operator.js

@@ -0,0 +1,44 @@
+/*\
+title: test-deserialize-operator.js
+type: application/javascript
+tags: [[$:/tags/test-spec]]
+
+Tests deserialize[] filter operator with various core deserializers
+
+\*/
+(function(){
+
+	/* jslint node: true, browser: true */
+	/* eslint-env node, browser, jasmine */
+	/* eslint no-mixed-spaces-and-tabs: ["error", "smart-tabs"]*/
+	/* global $tw, require */
+	"use strict";
+
+	
+	describe("deserialize operator tests", function() {
+
+		it("should support the deserialize[] operator", function() {
+			//Unknown deserializer as operand
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 4}deserialize[unknown/deserializer]]")).toEqual([$tw.language.getString("Error/DeserializeOperator/UnknownDeserializer")]);
+
+			//Missing operand
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 4}deserialize[]]")).toEqual([$tw.language.getString("Error/DeserializeOperator/MissingOperand")]);
+
+			//Deserialize TiddlyWiki file
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 4}deserialize[text/html]]")).toEqual(['[{"type":"text/vnd.tiddlywiki","text":"Abacus","title":"Hello \\"There\\""},{"title":"Hello \\"There\\"","text":"Calculator"}]']);
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 5}deserialize[text/html]]")).toEqual(['[{"type":"text/vnd.tiddlywiki","text":"Abacus","title":"Hello \\"There\\""},{"title":"Hello \\"There\\"","text":"Calculator"},{"title":"Hello \\"There\\"","text":"Protractor"}]']);
+
+			// Deserialize JSON payload containing tiddlers
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 6}deserialize[application/json]]")).toEqual( [ `[{"created":"20230601125557184","text":"Before you start storing important information in ~TiddlyWiki it is vital to make sure that you can reliably save changes. See https://tiddlywiki.com/#GettingStarted for details\\n\\n","title":"GettingStarted","modified":"20230601125601619"},{"created":"20230601125507054","text":"Welcome to \\"TiddlyWiki\\".\\n\\nThis is a test tiddler.","tags":"","title":"Hello There \\"Welcome\\"","modified":"20230601125551144"},{"title":"TiddlyWiki","created":"20130822170700000","modified":"20170127221451610","tags":"Concepts","type":"text/vnd.tiddlywiki","text":"~TiddlyWiki is a rich, interactive tool for manipulating complex data with structure that doesn't easily fit into conventional tools like spreadsheets or wordprocessors.\\n\\n~TiddlyWiki is designed to fit around your brain, helping you deal with the things that won't fit."}]` ]);
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 6}deserialize[application/json]jsonindexes[]] :map[{dezerializer test data case 6}jsonget<currentTiddler>,[title]]")).toEqual([ 'GettingStarted', 'Hello There "Welcome"', 'TiddlyWiki' ]);
+
+			//Deserialize TiddlyWiki file with an mismatched deserializer
+			expect($tw.wiki.filterTiddlers("[{dezerializer test data case 5}deserialize[application/json]]")).toEqual([jasmine.stringMatching('JSON error')]);
+		});
+	});
+	
+})();
+
+
+
+ 

editions/tw5.com/tiddlers/filters/deserialize Operator.tid

@@ -0,0 +1,26 @@
+caption: deserialize
+created: 20230601195749377
+from-version: 5.3.0
+modified: 20230602105513132
+op-input: a selection of strings
+op-output: JSON representations of tiddlers extracted from input titles.
+op-parameter: the deserializer module to be used to extract tiddlers from the input
+op-purpose: extract JSON representation of tiddlers from the input strings
+tags: [[Filter Operators]] [[Special Operators]]
+title: deserialize Operator
+type: text/vnd.tiddlywiki
+
+<<.tip "Deserializer modules parse text in various formats into their JSON representation as tiddlers. You can see the deserializers available in a wiki using the [[deserializers operator|deserializers Operator]].">>
+
+|!Deserializer |!Description |
+|(DOM)|Extracts tiddlers from a DOM node, should not be used with the <<.op deserialize[]>> operator |
+|application/javascript|Parses a JavaScript module as a tiddler extracting fields from the header comment|
+|application/json|Parses [[JSON|JSON in TiddlyWiki]] into tiddlers|
+|application/x-tiddler|Parses the [[.tid file format|TiddlerFiles]] as a tiddler|
+|application/x-tiddler-html-div|Parses the [[<DIV>.tiddler file format|TiddlerFiles]] as a tiddler|
+|application/x-tiddlers|Parses the [[MultiTiddlerFile format|MultiTiddlerFiles]] as tiddlers|
+|text/css|Parses CSS as a tiddler extracting fields from the header comment|
+|text/html|Parses an HTML file into tiddlers. Supports ~TiddlyWiki Classic HTML files, ~TiddlyWiki5 HTML files and ordinary HTML files|
+|text/plain|Parses plain text as a tiddler|
+
+<<.operator-examples "deserialize">>

editions/tw5.com/tiddlers/filters/examples/deserialize Operator (Examples).tid

@@ -0,0 +1,29 @@
+created: 20230601200356736
+modified: 20230602105036887
+tags: [[Operator Examples]] [[deserialize Operator]]
+title: deserialize Operator (Examples)
+type: text/vnd.tiddlywiki
+
+\define html-data()
+<!doctype html>
+<html lang="en">
+<head>
+	<meta charset="utf-8">
+	<title>Test Data</title>
+</head>
+<body>
+
+<!--~~ Ordinary tiddlers ~~-->
+<div id="storeArea" style="display:none;"><div title="Hello &quot;There&quot;" type="text/vnd.tiddlywiki">
+<pre>Abacus</pre>
+</div>
+</div>
+<script class="tiddlywiki-tiddler-store" type="application/json">[{"title":"Hello \"There\"","text":"Calculator"},{"title":"Hello \"There\"","text":"Protractor"}]</script>
+</body>
+</html>
+\end
+
+This example uses the predefined variable `html-data`:
+<$codeblock code=<<html-data>> language="HTML"/>
+
+<<.operator-example 1 "[<html-data>deserialize[text/html]]">>