From df286ecfe63def05587dbc6f34e18abc5ef85e59 Mon Sep 17 00:00:00 2001 From: "jeremy@jermolene.com" Date: Fri, 9 Dec 2022 18:28:12 +0000 Subject: [PATCH] Fixes for JSON operators --- core/modules/filters/json-ops.js | 61 +++++++++++++---- .../prerelease/tiddlers/Release 5.2.4.tid | 2 +- .../test/tiddlers/tests/test-json-filters.js | 29 ++++++-- .../tiddlers/features/JSON in TiddlyWiki.tid | 2 +- .../tw5.com/tiddlers/filters/jsonextract.tid | 66 +++++++++++++++++++ editions/tw5.com/tiddlers/filters/jsonget.tid | 15 +++-- .../tw5.com/tiddlers/filters/jsonindexes.tid | 1 + .../tw5.com/tiddlers/filters/jsontype.tid | 1 + .../Reading data from JSON tiddlers.tid | 11 +++- 9 files changed, 161 insertions(+), 27 deletions(-) create mode 100644 editions/tw5.com/tiddlers/filters/jsonextract.tid diff --git a/core/modules/filters/json-ops.js b/core/modules/filters/json-ops.js index d5e8c33af..eabf6433e 100644 --- a/core/modules/filters/json-ops.js +++ b/core/modules/filters/json-ops.js @@ -17,9 +17,23 @@ exports["jsonget"] = function(source,operator,options) { source(function(tiddler,title) { var data = $tw.utils.parseJSONSafe(title,title); if(data) { - var item = getDataItemValueAsString(data,operator.operands); + var items = getDataItemValueAsStrings(data,operator.operands); + if(items !== undefined) { + results.push.apply(results,items); + } + } + }); + return results; +}; + +exports["jsonextract"] = function(source,operator,options) { + var results = []; + source(function(tiddler,title) { + var data = $tw.utils.parseJSONSafe(title,title); + if(data) { + var item = getDataItem(data,operator.operands); if(item !== undefined) { - results.push(item); + results.push(JSON.stringify(item)); } } }); @@ -31,9 +45,9 @@ exports["jsonindexes"] = function(source,operator,options) { source(function(tiddler,title) { var data = $tw.utils.parseJSONSafe(title,title); if(data) { - var item = getDataItemKeysAsStrings(data,operator.operands); - if(item !== undefined) { - results.push.apply(results,item); + var items = getDataItemKeysAsStrings(data,operator.operands); + if(items !== undefined) { + results.push.apply(results,items); } } }); @@ -57,11 +71,11 @@ exports["jsontype"] = function(source,operator,options) { /* Given a JSON data structure and an array of index strings, return an array of the string representation of the values at the end of the index chain, or "undefined" if any of the index strings are invalid */ -function getDataItemValueAsString(data,indexes) { +function getDataItemValueAsStrings(data,indexes) { // Get the item var item = getDataItem(data,indexes); - // Return the item as a string - return convertDataItemValueToString(item); + // Return the item as a string list + return convertDataItemValueToStrings(item); } /* @@ -77,15 +91,34 @@ function getDataItemKeysAsStrings(data,indexes) { /* Return an array of the string representation of the values of a data item, or "undefined" if the item is undefined */ -function convertDataItemValueToString(item) { +function convertDataItemValueToStrings(item) { // Return the item as a string if(item === undefined) { - return item; + return undefined; + } else if(item === null) { + return ["null"] + } else if(typeof item === "object") { + var results = [],i,t; + if($tw.utils.isArray(item)) { + // Return all the items in arrays recursively + for(i=0; i> new GenesisWidget that allows the dynamic construction of another widget, where the name and attributes of the new widget can be dynamically determined, without needing to be known in advance -<<.link-badge-added "https://github.com/Jermolene/TiddlyWiki5/pull/6936">> new operators for reading and formatting JSON data: [[jsonget Operator]], [[jsonindexes Operator]], [[jsontype Operator]] and [[format Operator]] +<<.link-badge-added "https://github.com/Jermolene/TiddlyWiki5/pull/6936">> new operators for reading and formatting JSON data: [[jsonget Operator]], [[jsonindexes Operator]], [[jsontype Operator]], [[jsonextract Operator]] and [[format Operator]] ! Translation improvement diff --git a/editions/test/tiddlers/tests/test-json-filters.js b/editions/test/tiddlers/tests/test-json-filters.js index c892c2419..b7f4836d9 100644 --- a/editions/test/tiddlers/tests/test-json-filters.js +++ b/editions/test/tiddlers/tests/test-json-filters.js @@ -23,7 +23,7 @@ describe("json filter tests", function() { type: "application/json" },{ title: "Second", - text: '["une","deux","trois"]', + text: '["une","deux","trois",["quatre","cinq"]]', type: "application/json" },{ title: "Third", @@ -38,14 +38,14 @@ describe("json filter tests", function() { it("should support the jsonget operator", function() { expect(wiki.filterTiddlers("[{Third}jsonget[]]")).toEqual(["This is not JSON"]); - expect(wiki.filterTiddlers("[{First}jsonget[]]")).toEqual(['{"a":"one","b":"","c":1.618,"d":{"e":"four","f":["five","six",true,false,null]}}']); + expect(wiki.filterTiddlers("[{Second}jsonget[]]")).toEqual(["une","deux","trois","quatre","cinq"]); + expect(wiki.filterTiddlers("[{First}jsonget[]]")).toEqual(["one","","1.618","four","five","six","true","false","null"]); expect(wiki.filterTiddlers("[{First}jsonget[a]]")).toEqual(["one"]); expect(wiki.filterTiddlers("[{First}jsonget[b]]")).toEqual([""]); expect(wiki.filterTiddlers("[{First}jsonget[missing-property]]")).toEqual([]); - expect(wiki.filterTiddlers("[{First}jsonget[d]]")).toEqual(['{"e":"four","f":["five","six",true,false,null]}']); - expect(wiki.filterTiddlers("[{First}jsonget[d]jsonget[f]]")).toEqual(['["five","six",true,false,null]']); + expect(wiki.filterTiddlers("[{First}jsonget[d]]")).toEqual(["four","five","six","true","false","null"]); expect(wiki.filterTiddlers("[{First}jsonget[d],[e]]")).toEqual(["four"]); - expect(wiki.filterTiddlers("[{First}jsonget[d],[f]]")).toEqual(['["five","six",true,false,null]']); + expect(wiki.filterTiddlers("[{First}jsonget[d],[f]]")).toEqual(["five","six","true","false","null"]); expect(wiki.filterTiddlers("[{First}jsonget[d],[f],[0]]")).toEqual(["five"]); expect(wiki.filterTiddlers("[{First}jsonget[d],[f],[1]]")).toEqual(["six"]); expect(wiki.filterTiddlers("[{First}jsonget[d],[f],[2]]")).toEqual(["true"]); @@ -53,8 +53,25 @@ describe("json filter tests", function() { expect(wiki.filterTiddlers("[{First}jsonget[d],[f],[4]]")).toEqual(["null"]); }); + it("should support the jsonextract operator", function() { + expect(wiki.filterTiddlers("[{Third}jsonextract[]]")).toEqual(['"This is not JSON"']); + expect(wiki.filterTiddlers("[{First}jsonextract[]]")).toEqual(['{"a":"one","b":"","c":1.618,"d":{"e":"four","f":["five","six",true,false,null]}}']); + expect(wiki.filterTiddlers("[{First}jsonextract[a]]")).toEqual(['"one"']); + expect(wiki.filterTiddlers("[{First}jsonextract[b]]")).toEqual(['""']); + expect(wiki.filterTiddlers("[{First}jsonextract[missing-property]]")).toEqual([]); + expect(wiki.filterTiddlers("[{First}jsonextract[d]]")).toEqual(['{"e":"four","f":["five","six",true,false,null]}']); + expect(wiki.filterTiddlers("[{First}jsonextract[d]jsonextract[f]]")).toEqual(['["five","six",true,false,null]']); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[e]]")).toEqual(['"four"']); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f]]")).toEqual(['["five","six",true,false,null]']); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f],[0]]")).toEqual(['"five"']); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f],[1]]")).toEqual(['"six"']); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f],[2]]")).toEqual(["true"]); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f],[3]]")).toEqual(["false"]); + expect(wiki.filterTiddlers("[{First}jsonextract[d],[f],[4]]")).toEqual(["null"]); + }); + it("should support the jsonindexes operator", function() { - expect(wiki.filterTiddlers("[{Second}jsonindexes[]]")).toEqual(["0","1","2"]); + expect(wiki.filterTiddlers("[{Second}jsonindexes[]]")).toEqual(["0","1","2","3"]); expect(wiki.filterTiddlers("[{First}jsonindexes[]]")).toEqual(["a","b","c","d"]); expect(wiki.filterTiddlers("[{First}jsonindexes[a]]")).toEqual([]); expect(wiki.filterTiddlers("[{First}jsonindexes[b]]")).toEqual([]); diff --git a/editions/tw5.com/tiddlers/features/JSON in TiddlyWiki.tid b/editions/tw5.com/tiddlers/features/JSON in TiddlyWiki.tid index 2f2edd09f..cd827ddb6 100644 --- a/editions/tw5.com/tiddlers/features/JSON in TiddlyWiki.tid +++ b/editions/tw5.com/tiddlers/features/JSON in TiddlyWiki.tid @@ -2,7 +2,7 @@ title: JSON in TiddlyWiki tags: Features type: text/vnd.tiddlywiki created: 20220427174702859 -modified: 20220427174702859 +modified: 20220611104737314 !! Introduction diff --git a/editions/tw5.com/tiddlers/filters/jsonextract.tid b/editions/tw5.com/tiddlers/filters/jsonextract.tid new file mode 100644 index 000000000..15517e110 --- /dev/null +++ b/editions/tw5.com/tiddlers/filters/jsonextract.tid @@ -0,0 +1,66 @@ +created: 20220611104737314 +modified: 20220611104737314 +tags: [[Filter Operators]] [[JSON Operators]] +title: jsonextract Operator +caption: jsonextract +op-purpose: retrieve the JSON string of a property from JSON strings +op-input: a selection of JSON strings +op-parameter: one or more indexes of the property to retrieve +op-output: the JSON string values of each of the retrieved properties + +<<.from-version "5.2.4">> See [[JSON in TiddlyWiki]] for background. + +The <<.op jsonextract>> operator is used to retrieve values from JSON data as JSON substrings. See also the following related operators: + +* <<.olink jsonget>> to retrieve the values of a property in JSON data +* <<.olink jsontype>> to retrieve the type of a JSON value +* <<.olink jsonindexes>> to retrieve the names of the fields of a JSON object, or the indexes of a JSON array + +Properties within a JSON object are identified by a sequence of indexes. In the following example, the value at `[a]` is `one`, and the value at `[d][f][0]` is `five`. + +``` +{ + "a": "one", + "b": "", + "c": "three", + "d": { + "e": "four", + "f": [ + "five", + "six", + true, + false, + null + ], + "g": { + "x": "max", + "y": "may", + "z": "maize" + } + } +} +``` + +The following examples assume that this JSON data is contained in a variable called `jsondata`. + +The <<.op jsonextract>> operator uses multiple operands to specify the indexes of the property to retrieve. Values are returned as literal JSON strings: + +``` +[jsonextract[a]] --> "one" +[jsonextract[d],[e]] --> "four" +[jsonextract[d],[f],[0]] --> "five" +[jsonextract[d],[f]] --> ["five","six",true,false,null] +[jsonextract[d],[g]] --> {"x":"max","y":"may","z":"maize"} +``` + +Indexes can be dynamically composed from variables and transclusions: + +``` +[jsonextract,{!!field},[0]] +``` + +A subtlety is that the special case of a single blank operand is used to identify the root object. Thus: + +``` +[jsonextract[]] --> {"a":"one","b":"","c":"three","d":{"e":"four","f":["five","six",true,false,null],"g":{"x":"max","y":"may","z":"maize"}}} +``` diff --git a/editions/tw5.com/tiddlers/filters/jsonget.tid b/editions/tw5.com/tiddlers/filters/jsonget.tid index dbc247d7b..d9caa680e 100644 --- a/editions/tw5.com/tiddlers/filters/jsonget.tid +++ b/editions/tw5.com/tiddlers/filters/jsonget.tid @@ -10,10 +10,11 @@ op-output: the values of each of the retrieved properties <<.from-version "5.2.4">> See [[JSON in TiddlyWiki]] for background. -The <<.op jsonget>> operator is used to retrieve values from JSON data. See also the following related operators: +The <<.op jsonget>> operator is used to retrieve values from JSON data as strings. See also the following related operators: * <<.olink jsontype>> to retrieve the type of a JSON value * <<.olink jsonindexes>> to retrieve the names of the fields of a JSON object, or the indexes of a JSON array +* <<.olink jsonextract>> to retrieve a JSON value as a string of JSON Properties within a JSON object are identified by a sequence of indexes. In the following example, the value at `[a]` is `one`, and the value at `[d][f][0]` is `five`. @@ -65,11 +66,11 @@ Boolean values and null are returned as normal strings. The <<.olink jsontype>> [jsontype[d],[f],[2]] --> "boolean" ``` -Using the <<.op jsonget>> operator to retrieve an object or an array returns a JSON string of the values. For example: +Using the <<.op jsonget>> operator to retrieve an object or an array returns a list of the values. For example: ``` -[jsonget[d],[f]] --> `["five","six",true,false,null]` -[jsonget[d],[g]] --> `{"x": "max","y": "may","z": "maize"}` +[jsonget[d],[f]] --> "five","six","true","false","null" +[jsonget[d],[g]] --> "max","may","maize" ``` The <<.olink jsonindexes>> operator retrieves the corresponding indexes: @@ -79,6 +80,12 @@ The <<.olink jsonindexes>> operator retrieves the corresponding indexes: [jsonindexes[d],[g]] --> "x", "y", "z" ``` +If the object or array contains nested child objects or arrays then the values are retrieved recursively and returned flattened into a list. For example: + +``` +[jsonget[d]] --> "four","five","six","true","false","null","max","may","maize" +``` + A subtlety is that the special case of a single blank operand is used to identify the root object. Thus: ``` diff --git a/editions/tw5.com/tiddlers/filters/jsonindexes.tid b/editions/tw5.com/tiddlers/filters/jsonindexes.tid index 933f0f101..605936a2f 100644 --- a/editions/tw5.com/tiddlers/filters/jsonindexes.tid +++ b/editions/tw5.com/tiddlers/filters/jsonindexes.tid @@ -14,6 +14,7 @@ The <<.op jsonindexes>> operator is used to retrieve the property names of JSON * <<.olink jsonget>> to retrieve the values of a property in JSON data * <<.olink jsontype>> to retrieve the type of a JSON value +* <<.olink jsonextract>> to retrieve a JSON value as a string of JSON Properties within a JSON object are identified by a sequence of indexes. In the following example, the value at `[a]` is `one`, and the value at `[d][f][0]` is `five`. diff --git a/editions/tw5.com/tiddlers/filters/jsontype.tid b/editions/tw5.com/tiddlers/filters/jsontype.tid index 766757af0..b88f865dd 100644 --- a/editions/tw5.com/tiddlers/filters/jsontype.tid +++ b/editions/tw5.com/tiddlers/filters/jsontype.tid @@ -14,6 +14,7 @@ The <<.op jsontype>> operator is used to retrieve the type of a property in JSON * <<.olink jsonget>> to retrieve the values of a property in JSON data * <<.olink jsonindexes>> to retrieve the names of the fields of a JSON object, or the indexes of a JSON array +* <<.olink jsonextract>> to retrieve a JSON value as a string of JSON JSON supports the following data types: diff --git a/editions/tw5.com/tiddlers/howtos/Reading data from JSON tiddlers.tid b/editions/tw5.com/tiddlers/howtos/Reading data from JSON tiddlers.tid index acf482789..8f7968de0 100644 --- a/editions/tw5.com/tiddlers/howtos/Reading data from JSON tiddlers.tid +++ b/editions/tw5.com/tiddlers/howtos/Reading data from JSON tiddlers.tid @@ -1,11 +1,20 @@ created: 20220427174702859 -modified: 20220427171449102 +modified: 20220611104737314 tags: [[JSON in TiddlyWiki]] Learning title: Reading data from JSON tiddlers type: text/vnd.tiddlywiki See [[JSON in TiddlyWiki]] for an overview of using JSON in TiddlyWiki. +!! Filter Operators for Accessing JSON Data + +The following filter operators allow values to be read from JSON data: + +* <<.olink jsonget>> to retrieve the values of a property in JSON data +* <<.olink jsontype>> to retrieve the type of a JSON value +* <<.olink jsonindexes>> to retrieve the names of the fields of a JSON object, or the indexes of a JSON array +* <<.olink jsonextract>> to retrieve a JSON value as a string of JSON + !! Text References for Accessing JSON Data [[Text references|TextReference]] are a simple shortcut syntax to look up the value of a named property. For example, if a [[DictionaryTiddler|DictionaryTiddlers]] called `MonthDays` contains: