Add new "fetch" command

Like the load command except retrieves the file over HTTP/HTTPS.

Allows experimentation with server-side twederation

This is a cleaned up version of code that I wrote last year at TWEUM
2016 @inmysocks @pmario @twMat @xcazin

core/language/en-GB/Help/fetch.tid

@@ -0,0 +1,28 @@
+title: $:/language/Help/fetch
+description: Fetch tiddlers from wiki by URL
+
+Fetch one or more files over HTTP/HTTPS, and import the tiddlers matching a filter, optionally transforming the incoming titles.
+
+```
+--fetch file <url> <import-filter> <transform-filter>
+--fetch files <url-filter> <import-filter> <transform-filter>
+```
+
+With the "file" variant only a single file is fetched and the first parameter is the URL of the file to read.
+
+With the "files" variant, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
+
+The `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
+
+The `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
+
+Preceding the `--fetch` command with `--verbose` will output progress information during the import.
+
+Note that TiddlyWiki will not fetch an older version of an already loaded plugin.
+
+The following example retrieves all the non-system tiddlers from http://tiddlywiki.com and saves them to a JSON file:
+
+```
+tiddlywiki --verbose --fetch file "http://tiddlywiki.com/" "[!is[system]]" "" --rendertiddler "$:/core/templates/exporters/JsonFile" output.json text/plain "" exportFilter "[!is[system]]"
+```
+

core/modules/commands/fetch.js

@@ -0,0 +1,145 @@
+/*\
+title: $:/core/modules/commands/fetch.js
+type: application/javascript
+module-type: command
+
+Commands to fetch external tiddlers
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.info = {
+	name: "fetch",
+	synchronous: false
+};
+
+var Command = function(params,commander,callback) {
+	this.params = params;
+	this.commander = commander;
+	this.callback = callback;
+};
+
+Command.prototype.execute = function() {
+	if(this.params.length < 2) {
+		return "Missing subcommand and url";
+	}
+	var subcommand = this.params[0],
+		url = this.params[1],
+		importFilter = this.params[2] || "[all[tiddlers]]",
+		transformFilter = this.params[3] || "";
+	switch(subcommand) {
+		case "file":
+			return this.fetchFiles({
+				url: url,
+				importFilter: importFilter,
+				transformFilter: transformFilter,
+				callback: this.callback
+			});
+			break;
+		case "files":
+			return this.fetchFiles({
+				urlFilter: url,
+				importFilter: importFilter,
+				transformFilter: transformFilter,
+				callback: this.callback
+			});
+			break;
+	}
+	return null;
+};
+
+Command.prototype.fetchFiles = function(options) {
+	var self = this;
+	// Get the list of URLs
+	var urls;
+	if(options.url) {
+		urls = [options.url]
+	} else if(options.urlFilter) {
+		urls = $tw.wiki.filterTiddlers(options.urlFilter);
+	} else {
+		return "Missing URL";
+	}
+	// Process each URL in turn
+	var next = 0;
+	var getNextFile = function(err) {
+		if(err) {
+			return options.callback(err);
+		}
+		if(next < urls.length) {
+			self.fetchFile(urls[next++],options,getNextFile);
+		} else {
+			options.callback(null);
+		}
+	};
+	getNextFile(null);
+	// Success
+	return null;
+};
+
+Command.prototype.fetchFile = function(url,options,callback) {
+	var self = this,
+		lib = url.substr(0,8) === "https://" ? require("https") : require("http");
+	lib.get(url).on("response",function(response) {
+	    var type = (response.headers["content-type"] || "").split(";")[0],
+	    	body = "";
+	    self.commander.write("Reading " + url + ": ");
+	    response.on("data",function(chunk) {
+	        body += chunk;
+	        self.commander.write(".");
+	    });
+	    response.on("end",function() {
+	        self.commander.write("\n");
+	        if(response.statusCode === 200) {
+		        self.processBody(body,type,options);
+		        callback(null);
+	        } else {
+	        	callback("Error " + response.statusCode + " retrieving " + url)
+	        }
+	   	});
+	   	response.on("error",function(e) {
+			console.log("Error on GET request: " + e);
+			callback(e);
+	   	});
+	});
+	return null;
+};
+
+Command.prototype.processBody = function(body,type,options) {
+	// Deserialise the HTML file and put the tiddlers in their own wiki
+	var self = this,
+		incomingWiki = new $tw.Wiki(),
+		tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body,{});
+	$tw.utils.each(tiddlers,function(tiddler) {
+		incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
+	});
+	// Filter the tiddlers to select the ones we want
+	var filteredTitles = incomingWiki.filterTiddlers(options.importFilter);
+	// Import the selected tiddlers
+	var count = 0;
+	incomingWiki.each(function(tiddler,title) {
+		if(filteredTitles.indexOf(title) !== -1) {
+			var newTiddler;
+			if(options.transformFilter) {
+				var transformedTitle = (incomingWiki.filterTiddlers(options.transformFilter,null,self.commander.wiki.makeTiddlerIterator([title])) || [""])[0];
+				if(transformedTitle) {
+					self.commander.log("Importing " + title + " as " + transformedTitle)
+					newTiddler = new $tw.Tiddler(tiddler,{title: transformedTitle});
+				}
+			} else {
+				self.commander.log("Importing " + title)
+				newTiddler = tiddler;
+			}
+			self.commander.wiki.importTiddler(newTiddler);
+			count++;
+		}
+	});
+	self.commander.log("Imported " + count + " tiddlers")
+};
+
+exports.Command = Command;
+
+})();

editions/tw5.com/tiddlers/commands/FetchCommand.tid

@@ -0,0 +1,8 @@
+created: 20170218131511071
+modified: 20170218131511071
+tags: Commands
+title: FetchCommand
+type: text/vnd.tiddlywiki
+caption: fetch
+
+{{$:/language/Help/fetch}}