From 0d0ece63777a22422ab2ddae7dee996b28f62c5e Mon Sep 17 00:00:00 2001 From: Jermolene Date: Sat, 18 Feb 2017 13:33:41 +0000 Subject: [PATCH] Add new "fetch" command Like the load command except retrieves the file over HTTP/HTTPS. Allows experimentation with server-side twederation This is a cleaned up version of code that I wrote last year at TWEUM 2016 @inmysocks @pmario @twMat @xcazin --- core/language/en-GB/Help/fetch.tid | 28 ++++ core/modules/commands/fetch.js | 145 ++++++++++++++++++ .../tiddlers/commands/FetchCommand.tid | 8 + 3 files changed, 181 insertions(+) create mode 100644 core/language/en-GB/Help/fetch.tid create mode 100644 core/modules/commands/fetch.js create mode 100644 editions/tw5.com/tiddlers/commands/FetchCommand.tid diff --git a/core/language/en-GB/Help/fetch.tid b/core/language/en-GB/Help/fetch.tid new file mode 100644 index 000000000..54e6e5ec8 --- /dev/null +++ b/core/language/en-GB/Help/fetch.tid @@ -0,0 +1,28 @@ +title: $:/language/Help/fetch +description: Fetch tiddlers from wiki by URL + +Fetch one or more files over HTTP/HTTPS, and import the tiddlers matching a filter, optionally transforming the incoming titles. + +``` +--fetch file +--fetch files +``` + +With the "file" variant only a single file is fetched and the first parameter is the URL of the file to read. + +With the "files" variant, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs. + +The `` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided. + +The `` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title. + +Preceding the `--fetch` command with `--verbose` will output progress information during the import. + +Note that TiddlyWiki will not fetch an older version of an already loaded plugin. + +The following example retrieves all the non-system tiddlers from http://tiddlywiki.com and saves them to a JSON file: + +``` +tiddlywiki --verbose --fetch file "http://tiddlywiki.com/" "[!is[system]]" "" --rendertiddler "$:/core/templates/exporters/JsonFile" output.json text/plain "" exportFilter "[!is[system]]" +``` + diff --git a/core/modules/commands/fetch.js b/core/modules/commands/fetch.js new file mode 100644 index 000000000..8984d5e29 --- /dev/null +++ b/core/modules/commands/fetch.js @@ -0,0 +1,145 @@ +/*\ +title: $:/core/modules/commands/fetch.js +type: application/javascript +module-type: command + +Commands to fetch external tiddlers + +\*/ +(function(){ + +/*jslint node: true, browser: true */ +/*global $tw: false */ +"use strict"; + +exports.info = { + name: "fetch", + synchronous: false +}; + +var Command = function(params,commander,callback) { + this.params = params; + this.commander = commander; + this.callback = callback; +}; + +Command.prototype.execute = function() { + if(this.params.length < 2) { + return "Missing subcommand and url"; + } + var subcommand = this.params[0], + url = this.params[1], + importFilter = this.params[2] || "[all[tiddlers]]", + transformFilter = this.params[3] || ""; + switch(subcommand) { + case "file": + return this.fetchFiles({ + url: url, + importFilter: importFilter, + transformFilter: transformFilter, + callback: this.callback + }); + break; + case "files": + return this.fetchFiles({ + urlFilter: url, + importFilter: importFilter, + transformFilter: transformFilter, + callback: this.callback + }); + break; + } + return null; +}; + +Command.prototype.fetchFiles = function(options) { + var self = this; + // Get the list of URLs + var urls; + if(options.url) { + urls = [options.url] + } else if(options.urlFilter) { + urls = $tw.wiki.filterTiddlers(options.urlFilter); + } else { + return "Missing URL"; + } + // Process each URL in turn + var next = 0; + var getNextFile = function(err) { + if(err) { + return options.callback(err); + } + if(next < urls.length) { + self.fetchFile(urls[next++],options,getNextFile); + } else { + options.callback(null); + } + }; + getNextFile(null); + // Success + return null; +}; + +Command.prototype.fetchFile = function(url,options,callback) { + var self = this, + lib = url.substr(0,8) === "https://" ? require("https") : require("http"); + lib.get(url).on("response",function(response) { + var type = (response.headers["content-type"] || "").split(";")[0], + body = ""; + self.commander.write("Reading " + url + ": "); + response.on("data",function(chunk) { + body += chunk; + self.commander.write("."); + }); + response.on("end",function() { + self.commander.write("\n"); + if(response.statusCode === 200) { + self.processBody(body,type,options); + callback(null); + } else { + callback("Error " + response.statusCode + " retrieving " + url) + } + }); + response.on("error",function(e) { + console.log("Error on GET request: " + e); + callback(e); + }); + }); + return null; +}; + +Command.prototype.processBody = function(body,type,options) { + // Deserialise the HTML file and put the tiddlers in their own wiki + var self = this, + incomingWiki = new $tw.Wiki(), + tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body,{}); + $tw.utils.each(tiddlers,function(tiddler) { + incomingWiki.addTiddler(new $tw.Tiddler(tiddler)); + }); + // Filter the tiddlers to select the ones we want + var filteredTitles = incomingWiki.filterTiddlers(options.importFilter); + // Import the selected tiddlers + var count = 0; + incomingWiki.each(function(tiddler,title) { + if(filteredTitles.indexOf(title) !== -1) { + var newTiddler; + if(options.transformFilter) { + var transformedTitle = (incomingWiki.filterTiddlers(options.transformFilter,null,self.commander.wiki.makeTiddlerIterator([title])) || [""])[0]; + if(transformedTitle) { + self.commander.log("Importing " + title + " as " + transformedTitle) + newTiddler = new $tw.Tiddler(tiddler,{title: transformedTitle}); + } + } else { + self.commander.log("Importing " + title) + newTiddler = tiddler; + } + self.commander.wiki.importTiddler(newTiddler); + count++; + } + }); + self.commander.log("Imported " + count + " tiddlers") +}; + +exports.Command = Command; + +})(); diff --git a/editions/tw5.com/tiddlers/commands/FetchCommand.tid b/editions/tw5.com/tiddlers/commands/FetchCommand.tid new file mode 100644 index 000000000..8570d22ce --- /dev/null +++ b/editions/tw5.com/tiddlers/commands/FetchCommand.tid @@ -0,0 +1,8 @@ +created: 20170218131511071 +modified: 20170218131511071 +tags: Commands +title: FetchCommand +type: text/vnd.tiddlywiki +caption: fetch + +{{$:/language/Help/fetch}}