Add "raw-file(s)" options to "fetch" command

core/language/en-GB/Help/fetch.tid

@@ -6,15 +6,19 @@ Fetch one or more files over HTTP/HTTPS, and import the tiddlers matching a filt
 ```
 --fetch file <url> <import-filter> <transform-filter>
 --fetch files <url-filter> <import-filter> <transform-filter>
+--fetch raw-file <url> <transform-filter>
+--fetch raw-files <url-filter> <transform-filter>
 ```
 
-With the "file" variant only a single file is fetched and the first parameter is the URL of the file to read.
+The "file" and "files" variants fetch the specified files and attempt to import the tiddlers within them (the same processing as if the files were dragged into the browser window). The "raw-file" and "raw-files" variants fetch the specified files and then store the raw file data in tiddlers, without applying the import logic.
 
-With the "files" variant, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
+With the "file" and "raw-file" variants only a single file is fetched and the first parameter is the URL of the file to read.
 
-The `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
+With the "files" and "raw-files" variants, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
 
-The `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
+For the "file" and "files" variants, the `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
+
+For all variants, the `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
 
 Preceding the `--fetch` command with `--verbose` will output progress information during the import.
 
@@ -26,3 +30,9 @@ The following example retrieves all the non-system tiddlers from http://tiddlywi
 tiddlywiki --verbose --fetch file "http://tiddlywiki.com/" "[!is[system]]" "" --rendertiddler "$:/core/templates/exporters/JsonFile" output.json text/plain "" exportFilter "[!is[system]]"
 ```
 
+The following example retrieves the "favicon" file from tiddlywiki.com and saves it in a file called "output.ico". Note that the intermediate tiddler "Icon Tiddler" is quoted in the "--fetch" command because it is being used as a transformation filter to replace the default title, while there are no quotes for the "--savetiddler" command because it is being used directly as a title.
+
+```
+tiddlywiki --verbose --fetch raw-file "http://tiddlywiki.com/favicon.ico" "[[Icon Tiddler]]" --savetiddler "Icon Tiddler" output.ico
+```
+

core/modules/commands/fetch.js

@@ -27,24 +27,36 @@ Command.prototype.execute = function() {
 	if(this.params.length < 2) {
 		return "Missing subcommand and url";
 	}
-	var subcommand = this.params[0],
+	switch(this.params[0]) {
-		url = this.params[1],
+		case "raw-file":
-		importFilter = this.params[2] || "[all[tiddlers]]",
+			return this.fetchFiles({
-		transformFilter = this.params[3] || "";
+				raw: true,
-	switch(subcommand) {
+				url: this.params[1],
+				transformFilter: this.params[2] || "",
+				callback: this.callback
+			});
+			break;
 		case "file":
 			return this.fetchFiles({
-				url: url,
+				url: this.params[1],
-				importFilter: importFilter,
+				importFilter: this.params[2],
-				transformFilter: transformFilter,
+				transformFilter: this.params[3] || "",
+				callback: this.callback
+			});
+			break;
+		case "raw-files":
+			return this.fetchFiles({
+				raw: true,
+				urlFilter: this.params[1],
+				transformFilter: this.params[2] || "",
 				callback: this.callback
 			});
 			break;
 		case "files":
 			return this.fetchFiles({
-				urlFilter: url,
+				urlFilter: this.params[1],
-				importFilter: importFilter,
+				importFilter: this.params[2],
-				transformFilter: transformFilter,
+				transformFilter: this.params[3] || "",
 				callback: this.callback
 			});
 			break;
@@ -85,16 +97,16 @@ Command.prototype.fetchFile = function(url,options,callback) {
 		lib = url.substr(0,8) === "https://" ? require("https") : require("http");
 	lib.get(url).on("response",function(response) {
 	    var type = (response.headers["content-type"] || "").split(";")[0],
-	    	body = "";
+	    	data = [];
 	    self.commander.write("Reading " + url + ": ");
 	    response.on("data",function(chunk) {
-	        body += chunk;
+	        data.push(chunk);
 	        self.commander.write(".");
 	    });
 	    response.on("end",function() {
 	        self.commander.write("\n");
 	        if(response.statusCode === 200) {
-		        self.processBody(body,type,options);
+		        self.processBody(Buffer.concat(data),type,options,url);
 		        callback(null);
 	        } else {
 	        	callback("Error " + response.statusCode + " retrieving " + url)
@@ -108,16 +120,27 @@ Command.prototype.fetchFile = function(url,options,callback) {
 	return null;
 };
 
-Command.prototype.processBody = function(body,type,options) {
+Command.prototype.processBody = function(body,type,options,url) {
-	// Deserialise the HTML file and put the tiddlers in their own wiki
+	var self = this;
-	var self = this,
+	// Collect the tiddlers in a wiki
-		incomingWiki = new $tw.Wiki(),
+	var incomingWiki = new $tw.Wiki();
-		tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body,{});
+	if(options.raw) {
-	$tw.utils.each(tiddlers,function(tiddler) {
+		var typeInfo = type ? $tw.config.contentTypeInfo[type] : null,
-		incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
+			encoding = typeInfo ? typeInfo.encoding : "utf8";
-	});
+		incomingWiki.addTiddler(new $tw.Tiddler({
+			title: url,
+			type: type,
+			text: body.toString(encoding)
+		}));
+	} else {
+		// Deserialise the file to extract the tiddlers
+		var tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body.toString("utf8"),{});
+		$tw.utils.each(tiddlers,function(tiddler) {
+			incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
+		});
+	}
 	// Filter the tiddlers to select the ones we want
-	var filteredTitles = incomingWiki.filterTiddlers(options.importFilter);
+	var filteredTitles = incomingWiki.filterTiddlers(options.importFilter || "[all[tiddlers]]");
 	// Import the selected tiddlers
 	var count = 0;
 	incomingWiki.each(function(tiddler,title) {