mirror of
https://github.com/Jermolene/TiddlyWiki5
synced 2024-06-25 23:03:15 +00:00
Add new "fetch" command
Like the load command except retrieves the file over HTTP/HTTPS. Allows experimentation with server-side twederation This is a cleaned up version of code that I wrote last year at TWEUM 2016 @inmysocks @pmario @twMat @xcazin
This commit is contained in:
parent
74b0da065e
commit
0d0ece6377
28
core/language/en-GB/Help/fetch.tid
Normal file
28
core/language/en-GB/Help/fetch.tid
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
title: $:/language/Help/fetch
|
||||||
|
description: Fetch tiddlers from wiki by URL
|
||||||
|
|
||||||
|
Fetch one or more files over HTTP/HTTPS, and import the tiddlers matching a filter, optionally transforming the incoming titles.
|
||||||
|
|
||||||
|
```
|
||||||
|
--fetch file <url> <import-filter> <transform-filter>
|
||||||
|
--fetch files <url-filter> <import-filter> <transform-filter>
|
||||||
|
```
|
||||||
|
|
||||||
|
With the "file" variant only a single file is fetched and the first parameter is the URL of the file to read.
|
||||||
|
|
||||||
|
With the "files" variant, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
|
||||||
|
|
||||||
|
The `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
|
||||||
|
|
||||||
|
The `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
|
||||||
|
|
||||||
|
Preceding the `--fetch` command with `--verbose` will output progress information during the import.
|
||||||
|
|
||||||
|
Note that TiddlyWiki will not fetch an older version of an already loaded plugin.
|
||||||
|
|
||||||
|
The following example retrieves all the non-system tiddlers from http://tiddlywiki.com and saves them to a JSON file:
|
||||||
|
|
||||||
|
```
|
||||||
|
tiddlywiki --verbose --fetch file "http://tiddlywiki.com/" "[!is[system]]" "" --rendertiddler "$:/core/templates/exporters/JsonFile" output.json text/plain "" exportFilter "[!is[system]]"
|
||||||
|
```
|
||||||
|
|
145
core/modules/commands/fetch.js
Normal file
145
core/modules/commands/fetch.js
Normal file
|
@ -0,0 +1,145 @@
|
||||||
|
/*\
|
||||||
|
title: $:/core/modules/commands/fetch.js
|
||||||
|
type: application/javascript
|
||||||
|
module-type: command
|
||||||
|
|
||||||
|
Commands to fetch external tiddlers
|
||||||
|
|
||||||
|
\*/
|
||||||
|
(function(){
|
||||||
|
|
||||||
|
/*jslint node: true, browser: true */
|
||||||
|
/*global $tw: false */
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
exports.info = {
|
||||||
|
name: "fetch",
|
||||||
|
synchronous: false
|
||||||
|
};
|
||||||
|
|
||||||
|
var Command = function(params,commander,callback) {
|
||||||
|
this.params = params;
|
||||||
|
this.commander = commander;
|
||||||
|
this.callback = callback;
|
||||||
|
};
|
||||||
|
|
||||||
|
Command.prototype.execute = function() {
|
||||||
|
if(this.params.length < 2) {
|
||||||
|
return "Missing subcommand and url";
|
||||||
|
}
|
||||||
|
var subcommand = this.params[0],
|
||||||
|
url = this.params[1],
|
||||||
|
importFilter = this.params[2] || "[all[tiddlers]]",
|
||||||
|
transformFilter = this.params[3] || "";
|
||||||
|
switch(subcommand) {
|
||||||
|
case "file":
|
||||||
|
return this.fetchFiles({
|
||||||
|
url: url,
|
||||||
|
importFilter: importFilter,
|
||||||
|
transformFilter: transformFilter,
|
||||||
|
callback: this.callback
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
case "files":
|
||||||
|
return this.fetchFiles({
|
||||||
|
urlFilter: url,
|
||||||
|
importFilter: importFilter,
|
||||||
|
transformFilter: transformFilter,
|
||||||
|
callback: this.callback
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
Command.prototype.fetchFiles = function(options) {
|
||||||
|
var self = this;
|
||||||
|
// Get the list of URLs
|
||||||
|
var urls;
|
||||||
|
if(options.url) {
|
||||||
|
urls = [options.url]
|
||||||
|
} else if(options.urlFilter) {
|
||||||
|
urls = $tw.wiki.filterTiddlers(options.urlFilter);
|
||||||
|
} else {
|
||||||
|
return "Missing URL";
|
||||||
|
}
|
||||||
|
// Process each URL in turn
|
||||||
|
var next = 0;
|
||||||
|
var getNextFile = function(err) {
|
||||||
|
if(err) {
|
||||||
|
return options.callback(err);
|
||||||
|
}
|
||||||
|
if(next < urls.length) {
|
||||||
|
self.fetchFile(urls[next++],options,getNextFile);
|
||||||
|
} else {
|
||||||
|
options.callback(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
getNextFile(null);
|
||||||
|
// Success
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
Command.prototype.fetchFile = function(url,options,callback) {
|
||||||
|
var self = this,
|
||||||
|
lib = url.substr(0,8) === "https://" ? require("https") : require("http");
|
||||||
|
lib.get(url).on("response",function(response) {
|
||||||
|
var type = (response.headers["content-type"] || "").split(";")[0],
|
||||||
|
body = "";
|
||||||
|
self.commander.write("Reading " + url + ": ");
|
||||||
|
response.on("data",function(chunk) {
|
||||||
|
body += chunk;
|
||||||
|
self.commander.write(".");
|
||||||
|
});
|
||||||
|
response.on("end",function() {
|
||||||
|
self.commander.write("\n");
|
||||||
|
if(response.statusCode === 200) {
|
||||||
|
self.processBody(body,type,options);
|
||||||
|
callback(null);
|
||||||
|
} else {
|
||||||
|
callback("Error " + response.statusCode + " retrieving " + url)
|
||||||
|
}
|
||||||
|
});
|
||||||
|
response.on("error",function(e) {
|
||||||
|
console.log("Error on GET request: " + e);
|
||||||
|
callback(e);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
Command.prototype.processBody = function(body,type,options) {
|
||||||
|
// Deserialise the HTML file and put the tiddlers in their own wiki
|
||||||
|
var self = this,
|
||||||
|
incomingWiki = new $tw.Wiki(),
|
||||||
|
tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body,{});
|
||||||
|
$tw.utils.each(tiddlers,function(tiddler) {
|
||||||
|
incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
|
||||||
|
});
|
||||||
|
// Filter the tiddlers to select the ones we want
|
||||||
|
var filteredTitles = incomingWiki.filterTiddlers(options.importFilter);
|
||||||
|
// Import the selected tiddlers
|
||||||
|
var count = 0;
|
||||||
|
incomingWiki.each(function(tiddler,title) {
|
||||||
|
if(filteredTitles.indexOf(title) !== -1) {
|
||||||
|
var newTiddler;
|
||||||
|
if(options.transformFilter) {
|
||||||
|
var transformedTitle = (incomingWiki.filterTiddlers(options.transformFilter,null,self.commander.wiki.makeTiddlerIterator([title])) || [""])[0];
|
||||||
|
if(transformedTitle) {
|
||||||
|
self.commander.log("Importing " + title + " as " + transformedTitle)
|
||||||
|
newTiddler = new $tw.Tiddler(tiddler,{title: transformedTitle});
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
self.commander.log("Importing " + title)
|
||||||
|
newTiddler = tiddler;
|
||||||
|
}
|
||||||
|
self.commander.wiki.importTiddler(newTiddler);
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
self.commander.log("Imported " + count + " tiddlers")
|
||||||
|
};
|
||||||
|
|
||||||
|
exports.Command = Command;
|
||||||
|
|
||||||
|
})();
|
8
editions/tw5.com/tiddlers/commands/FetchCommand.tid
Normal file
8
editions/tw5.com/tiddlers/commands/FetchCommand.tid
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
created: 20170218131511071
|
||||||
|
modified: 20170218131511071
|
||||||
|
tags: Commands
|
||||||
|
title: FetchCommand
|
||||||
|
type: text/vnd.tiddlywiki
|
||||||
|
caption: fetch
|
||||||
|
|
||||||
|
{{$:/language/Help/fetch}}
|
Loading…
Reference in New Issue
Block a user