mirrors
/
TiddlyWiki5
mirror of https://github.com/Jermolene/TiddlyWiki5
1
0
Fork 0
Code Issues Releases Wiki Activity

Add "raw-file(s)" options to "fetch" command

This commit is contained in:
Jermolene 2017-07-03 16:52:10 +01:00
parent 4db950cc45
commit c179dc93cb
2 changed files with 60 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
core/language/en-GB/Help/fetch.tid
View File

@ -6,15 +6,19 @@ Fetch one or more files over HTTP/HTTPS, and import the tiddlers matching a filt
```
--fetch file <url> <import-filter> <transform-filter>
--fetch files <url-filter> <import-filter> <transform-filter>
--fetch raw-file <url> <transform-filter>
--fetch raw-files <url-filter> <transform-filter>
```
With the "file" variant only a single file is fetched and the first parameter is the URL of the file to read.
The "file" and "files" variants fetch the specified files and attempt to import the tiddlers within them (the same processing as if the files were dragged into the browser window). The "raw-file" and "raw-files" variants fetch the specified files and then store the raw file data in tiddlers, without applying the import logic.
With the "files" variant, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
With the "file" and "raw-file" variants only a single file is fetched and the first parameter is the URL of the file to read.
The `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
With the "files" and "raw-files" variants, multiple files are fetched and the first parameter is a filter yielding a list of URLs of the files to read. For example, given a set of tiddlers tagged "remote-server" that have a field "url" the filter `[tag[remote-server]get[url]]` will retrieve all the available URLs.
The `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
For the "file" and "files" variants, the `<import-filter>` parameter specifies a filter determining which tiddlers are imported. It defaults to `[all[tiddlers]]` if not provided.
For all variants, the `<transform-filter>` parameter specifies an optional filter that transforms the titles of the imported tiddlers. For example, `[addprefix[$:/myimports/]]` would add the prefix `$:/myimports/` to each title.
Preceding the `--fetch` command with `--verbose` will output progress information during the import.
@ -26,3 +30,9 @@ The following example retrieves all the non-system tiddlers from http://tiddlywi
tiddlywiki --verbose --fetch file "http://tiddlywiki.com/" "[!is[system]]" "" --rendertiddler "$:/core/templates/exporters/JsonFile" output.json text/plain "" exportFilter "[!is[system]]"
```
The following example retrieves the "favicon" file from tiddlywiki.com and saves it in a file called "output.ico". Note that the intermediate tiddler "Icon Tiddler" is quoted in the "--fetch" command because it is being used as a transformation filter to replace the default title, while there are no quotes for the "--savetiddler" command because it is being used directly as a title.
```
tiddlywiki --verbose --fetch raw-file "http://tiddlywiki.com/favicon.ico" "[[Icon Tiddler]]" --savetiddler "Icon Tiddler" output.ico
```

69
core/modules/commands/fetch.js
View File

@ -27,24 +27,36 @@ Command.prototype.execute = function() {
if(this.params.length < 2) {
return "Missing subcommand and url";
}
var subcommand = this.params[0],
url = this.params[1],
importFilter = this.params[2] || "[all[tiddlers]]",
transformFilter = this.params[3] || "";
switch(subcommand) {
switch(this.params[0]) {
case "raw-file":
return this.fetchFiles({
raw: true,
url: this.params[1],
transformFilter: this.params[2] || "",
callback: this.callback
});
break;
case "file":
return this.fetchFiles({
url: url,
importFilter: importFilter,
transformFilter: transformFilter,
url: this.params[1],
importFilter: this.params[2],
transformFilter: this.params[3] || "",
callback: this.callback
});
break;
case "raw-files":
return this.fetchFiles({
raw: true,
urlFilter: this.params[1],
transformFilter: this.params[2] || "",
callback: this.callback
});
break;
case "files":
return this.fetchFiles({
urlFilter: url,
importFilter: importFilter,
transformFilter: transformFilter,
urlFilter: this.params[1],
importFilter: this.params[2],
transformFilter: this.params[3] || "",
callback: this.callback
});
break;
@ -85,16 +97,16 @@ Command.prototype.fetchFile = function(url,options,callback) {
lib = url.substr(0,8) === "https://" ? require("https") : require("http");
lib.get(url).on("response",function(response) {
var type = (response.headers["content-type"] || "").split(";")[0],
body = "";
data = [];
self.commander.write("Reading " + url + ": ");
response.on("data",function(chunk) {
body += chunk;
data.push(chunk);
self.commander.write(".");
});
response.on("end",function() {
self.commander.write("\n");
if(response.statusCode === 200) {
self.processBody(body,type,options);
self.processBody(Buffer.concat(data),type,options,url);
callback(null);
} else {
callback("Error " + response.statusCode + " retrieving " + url)
@ -108,16 +120,27 @@ Command.prototype.fetchFile = function(url,options,callback) {
return null;
};
Command.prototype.processBody = function(body,type,options) {
// Deserialise the HTML file and put the tiddlers in their own wiki
var self = this,
incomingWiki = new $tw.Wiki(),
tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body,{});
$tw.utils.each(tiddlers,function(tiddler) {
incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
});
Command.prototype.processBody = function(body,type,options,url) {
var self = this;
// Collect the tiddlers in a wiki
var incomingWiki = new $tw.Wiki();
if(options.raw) {
var typeInfo = type ? $tw.config.contentTypeInfo[type] : null,
encoding = typeInfo ? typeInfo.encoding : "utf8";
incomingWiki.addTiddler(new $tw.Tiddler({
title: url,
type: type,
text: body.toString(encoding)
}));
} else {
// Deserialise the file to extract the tiddlers
var tiddlers = this.commander.wiki.deserializeTiddlers(type || "text/html",body.toString("utf8"),{});
$tw.utils.each(tiddlers,function(tiddler) {
incomingWiki.addTiddler(new $tw.Tiddler(tiddler));
});
}
// Filter the tiddlers to select the ones we want
var filteredTitles = incomingWiki.filterTiddlers(options.importFilter);
var filteredTitles = incomingWiki.filterTiddlers(options.importFilter || "[all[tiddlers]]");
// Import the selected tiddlers
var count = 0;
incomingWiki.each(function(tiddler,title) {