tm-http-request: Add support for binary responses

With a demo courtesy of https://random.dog/

@rmunn you recently worked on the base64 utilities. I tried to use $tw.utils.base64Encode instead of window.btoa, but found that it didn't work. It's concerning because we expose that utility method as a filter operation, and it would be frustrating if we were not base64encoding things properly.

core/modules/startup/rootwidget.js

@@ -38,6 +38,7 @@ exports.startup = function() {
 			url: params.url,
 			method: params.method,
 			body: params.body,
+			binary: params.binary,
 			oncompletion: params.oncompletion,
 			onprogress: params.onprogress,
 			bindStatus: params["bind-status"],

core/modules/utils/dom/http.js

@@ -90,6 +90,7 @@ wiki: wiki to be used for executing action strings
 url: URL for request
 method: method eg GET, POST
 body: text of request body
+binary: set to "yes" to force binary processing of response payload
 oncompletion: action string to be invoked on completion
 onprogress: action string to be invoked on progress updates
 bindStatus: optional title of tiddler to which status ("pending", "complete", "error") should be written
@@ -110,6 +111,7 @@ function HttpClientRequest(options) {
 	this.bindProgress = options["bindProgress"];
 	this.method = options.method || "GET";
 	this.body = options.body || "";
+	this.binary = options.binary || "";
 	this.variables = options.variables;
 	var url = options.url;
 	$tw.utils.each(options.queryStrings,function(value,name) {
@@ -156,6 +158,8 @@ HttpClientRequest.prototype.send = function(callback) {
 			type: this.method,
 			headers: this.requestHeaders,
 			data: this.body,
+			returnProp: this.binary === "" ? "responseText" : "response",
+			responseType: this.binary === "" ? "text" : "arraybuffer",
 			callback: function(err,data,xhr) {
 				var hasSucceeded = xhr.status >= 200 && xhr.status < 300,
 					completionCode = hasSucceeded ? "complete" : "error",
@@ -175,6 +179,16 @@ HttpClientRequest.prototype.send = function(callback) {
 					data: (data || "").toString(),
 					headers: JSON.stringify(headers)
 				};
+				/* Convert data from binary to base64 */
+				if (xhr.responseType === "arraybuffer") {
+					var binary = "",
+						bytes = new Uint8Array(data),
+						len = bytes.byteLength;
+					for (var i=0; i<len; i++) {
+						binary += String.fromCharCode(bytes[i]);
+					}
+					resultVariables.data = window.btoa(binary);
+				}
 				self.wiki.addTiddler(new $tw.Tiddler(self.wiki.getTiddler(requestTrackerTitle),{
 					status: completionCode,
 				}));
@@ -212,6 +226,7 @@ Make an HTTP request. Options are:
 	callback: function invoked with (err,data,xhr)
 	progress: optional function invoked with (lengthComputable,loaded,total)
 	returnProp: string name of the property to return as first argument of callback
+	responseType: "text" or "arraybuffer"
 */
 exports.httpRequest = function(options) {
 	var type = options.type || "GET",
@@ -264,6 +279,7 @@ exports.httpRequest = function(options) {
 			}
 		}
 	}
+	request.responseType = options.responseType || "text";
 	// Set up the state change handler
 	request.onreadystatechange = function() {
 		if(this.readyState === 4) {

editions/tw5.com/tiddlers/messages/WidgetMessage_ tm-http-request Example Random Dog.tid

@@ -0,0 +1,88 @@
+title: WidgetMessage: tm-http-request Example - Random Dog
+tags: $:/tags/Global
+
+\procedure download-dog(url)
+
+\procedure completion-download-dog()
+\import [subfilter{$:/core/config/GlobalImportFilter}]
+	<$action-log msg="In completion-download-dog"/>
+	<$action-log/>
+	<!-- Success -->
+	<$list filter="[<status>compare:number:gteq[200]compare:number:lteq[299]]" variable="ignore">
+		<!-- Create the dog tiddler -->
+		<$action-createtiddler
+			$basetitle=`$:/RandomDog/$(title)$`
+			text=<<data>>
+			tags="$:/tags/RandomDog"
+			type={{{ [<headers>jsonget[content-type]] }}}
+			credits="https://random.dog/"
+		>
+			<$action-log msg="Created tiddler" title=<<createTiddler-title>>/>
+		</$createtiddler>
+	</$list>
+\end completion-download-dog
+
+<$action-sendmessage
+	$message="tm-http-request"
+	url=<<url>>
+	method="GET"
+	binary="yes"
+	oncompletion=<<completion-download-dog>>
+	var-title=<<url>>
+/>
+\end download-dog
+
+\procedure get-random-dog()
+
+\procedure completion-get-json()
+\import [subfilter{$:/core/config/GlobalImportFilter}]
+	<$action-log msg="In completion-get-json"/>
+	<$action-log/>
+	<!-- Success -->
+	<$list filter="[<status>compare:number:gteq[200]compare:number:lteq[299]]" variable="ignore">
+		<!-- Download the dog -->
+		<$macrocall $name="download-dog" url={{{ [<data>jsonget[url]] }}}/>
+	</$list>
+\end completion-get-json
+
+<$action-sendmessage
+	$message="tm-http-request"
+	url="https://random.dog/woof.json"
+	method="GET"
+	oncompletion=<<completion-get-json>>
+/>
+\end get-random-dog
+
+!! Random Dogs
+
+This demo uses the API of the website https://random.dog/ to import a random dog image or video.
+
+<$button actions=<<get-random-dog>>>
+Import a random dog image or video
+</$button>
+
+<$list filter="[tag[$:/tags/RandomDog]limit[1]]" variable="ignore">
+
+!! Imported Tiddlers
+
+<$button>
+<$action-deletetiddler $filter="[tag[$:/tags/RandomDog]]"/>
+Delete all imported random dogs
+</$button>
+
+Export all imported random dogs: <$macrocall $name="exportButton" exportFilter="[tag[$:/tags/RandomDog]]" lingoBase="$:/language/Buttons/ExportTiddlers/"/>
+
+</$list>
+
+<ol>
+<$list filter="[tag[$:/tags/RandomDog]!sort[modified]]">
+<li>
+<$link>
+<$text text=<<currentTiddler>>/>
+</$link>
+<div style="width:300px;height:300px;">
+<$transclude $tiddler=<<currentTiddler>>/>
+</div>
+</li>
+</$list>
+</ol>

editions/tw5.com/tiddlers/messages/WidgetMessage_ tm-http-request Example Zotero.tid

@@ -1,5 +1,5 @@
 title: WidgetMessage: tm-http-request Example - Zotero
-tags: $:/tags/Macro
+tags: $:/tags/Global
 
 \procedure select-zotero-group()
 Specify the Zotero group ID to import
@@ -34,7 +34,7 @@ Specify the Zotero group ID to import
 \procedure zotero-get-items(start:"0",limit:"25")
 
 \procedure completion()
-\import [[$:/core/ui/PageMacros]] [all[shadows+tiddlers]tag[$:/tags/Macro]!has[draft.of]]
+\import [subfilter{$:/core/config/GlobalImportFilter}]
 	<$action-log msg="In completion"/>
 	<$action-log/>
 	<!-- Success -->

editions/tw5.com/tiddlers/messages/WidgetMessage_ tm-http-request.tid

@@ -1,6 +1,6 @@
 caption: tm-http-request
 created: 20230429161453032
-modified: 20230429161453032
+modified: 20230717104212742
 tags: Messages
 title: WidgetMessage: tm-http-request
 type: text/vnd.tiddlywiki
@@ -18,6 +18,7 @@ The following parameters are used:
 |!Name |!Description |
 |method |HTTP method (eg "GET", "POST") |
 |body |String data to be sent with the request |
+|binary |<<.from-version "5.3.1">> Set to "yes" to cause the response body to be treated as binary data and returned in base64 format |
 |query-* |Query string parameters with string values |
 |header-* |Headers with string values |
 |password-header-* |Headers with values taken from the password store |
@@ -49,3 +50,4 @@ Note that the state tiddler $:/state/http-requests contains a number representin
 !! Examples
 
 * [[Zotero's|https://www.zotero.org/]] API for retrieving reference items: [[WidgetMessage: tm-http-request Example - Zotero]]
+* [[Random Dog's|https://random.dog/]] API for retrieving random pictures of dogs showing how to retrieve binary data: [[WidgetMessage: tm-http-request Example - Random Dogs]]