diff --git a/core/modules/commands/exec.js b/core/modules/commands/exec.js new file mode 100644 index 000000000..f8fa87b4b --- /dev/null +++ b/core/modules/commands/exec.js @@ -0,0 +1,121 @@ +/*\ +title: $:/core/modules/commands/exec.js +type: application/javascript +module-type: command + +Command to execute an external task + +\*/ +(function(){ + +/*jslint node: true, browser: true */ +/*global $tw: false */ +"use strict"; + +exports.info = { + name: "exec", + synchronous: false +}; + +var Command = function(params,commander,callback) { + this.params = params; + this.commander = commander; + this.callback = callback; +}; + +Command.prototype.execute = function() { + var self = this; + if(this.params.length < 2) { + return "Missing parameters"; + } + var name = self.params[0], // External task name + filter = self.params[1], // Filter of tiddlers to pass to the task + args = self.params.slice(2); // Remaining arguments are passed to the task arguments + // Find the task information + var taskInfo = ($tw.boot.wikiInfo["external-tasks"] || {})[name]; + if(!taskInfo) { + return this.callback("External task \"" + name + "\" not found"); + } + // Execute it + var spawn = require("child_process").spawn, + path = require("path"), + childProcess = spawn(path.resolve($tw.boot.wikiPath,taskInfo.path),args,{ + stdio: ["pipe","pipe",process.stderr], + shell: true, + env: $tw.utils.extend({},process.env,taskInfo.environment) + }); + // Choose the input representation + var taskInfoInput = taskInfo.input || {}, + data; + switch(taskInfoInput.format || "json-raw-tiddlers") { + case "rendered-text": + var titles = self.commander.wiki.filterTiddlers(filter), + output = []; + $tw.utils.each(titles,function(title) { + output.push(self.commander.wiki.renderTiddler("text/plain",title)); + }); + data = output.join(""); + break; + case "json-rendered-text-tiddlers": + var titles = self.commander.wiki.filterTiddlers(filter), + tiddlers = []; + $tw.utils.each(titles,function(title) { + tiddlers.push({ + title: title, + text: self.commander.wiki.renderTiddler("text/plain",title) + }) + }); + data = JSON.stringify(tiddlers); + break; + case "json-raw-tiddlers": + // Intentional fall-through + default: + data = this.commander.wiki.getTiddlersAsJson(filter); + break; + } + if(data === undefined) { + return this.callback("No input data defined for tiddler processor"); + } + // Pass the tiddlers as JSON over stdin + childProcess.stdin.write(data); + childProcess.stdin.end(); + // Catch the output + var chunks = []; + childProcess.stdout.on("data",function(chunk) { + chunks.push(chunk.toString()); + }); + // Pick up the output when the process ends + childProcess.once("exit",function(code) { + if(code !== 0) { + return self.callback("Error executing external task: " + code); + } + var childOutput = chunks.join(""), + taskInfoOutput = taskInfo.output || {}, + data; + switch(taskInfoOutput.format || "text") { + case "json-raw-tiddlers": + try { + data = JSON.parse(childOutput); + } catch(e) { + self.callback("Error parsing returned JSON: " + e + "\n\n\n->\n" + childOutput); + } + // Add the tiddlers + self.commander.wiki.addTiddlers(data); + break; + case "text": + // Intentional fall-through + default: + self.commander.wiki.addTiddler(new $tw.Tiddler(taskInfoOutput.tiddler,{ + text: childOutput + })); + break; + } + // Exit successfully + self.callback(null); + }); + return null; +}; + +exports.Command = Command; + +})(); diff --git a/editions/dev/tiddlers/HelloThere.tid b/editions/dev/tiddlers/HelloThere.tid index 83dd24d04..8a4aba98b 100644 --- a/editions/dev/tiddlers/HelloThere.tid +++ b/editions/dev/tiddlers/HelloThere.tid @@ -1,6 +1,8 @@ -modified: 20141122200310516 +created: 20181001170605910 +modified: 20181001171639702 tags: TableOfContents title: HelloThere +type: text/vnd.tiddlywiki Welcome to the developer documentation for TiddlyWiki (https://tiddlywiki.com/). It is currently a work in progress as material from two different sources is adapted and merged in addition to original content being added: @@ -22,5 +24,6 @@ Welcome to the developer documentation for TiddlyWiki (https://tiddlywiki.com/). ** SyncAdaptorModules ** WidgetModules ** WikiRuleModules -*Original developer documentation +* New developer documentation ** HookMechanism +** [[External Shell Tasks]] diff --git a/editions/dev/tiddlers/new/External Shell Tasks.tid b/editions/dev/tiddlers/new/External Shell Tasks.tid new file mode 100644 index 000000000..90115a58c --- /dev/null +++ b/editions/dev/tiddlers/new/External Shell Tasks.tid @@ -0,0 +1,131 @@ +created: 20181001171604072 +modified: 20181001184306738 +title: External Shell Tasks +type: text/vnd.tiddlywiki + +!! Introduction + +It can be difficult for experienced developers to become proficient in working with TiddlyWiki's code: besides requiring a good understanding of JavaScript, the internals are built around relatively unusual concepts and techniques that are likely to be unfamiliar to most. + +However, many software developers are very familiar with using and creating text-based command line tools that adhere to the Unix philosophy of using stdin/stdout for input and output, allowing tools to be chained together in powerful ways. + +The external task mechanism allows Unix-style tasks to be registered and executed by TiddlyWiki under Node.js. Execution can be triggered via a server command or via a message propagated from the browser. + +!! Configuration + +External tasks must be registered in the `tiddlywiki.info` file of a wiki folder. For example: + +``` +{ + "description": "Edition demonstrating external tasks", + "plugins": [ + ], + "themes": [ + "tiddlywiki/vanilla", + "tiddlywiki/snowwhite" + ], + "includeWikis": [ + "../tw5.com" + ], + "external-tasks": { + "stats": { + "path": "./demo-tasks/stats.js", + "input": { + "format": "json-raw-tiddlers" + }, + "environment": { + "MY_VARIABLE": "value" + }, + "output": { + "format": "json-raw-tiddlers" + }, + "timeout": 100 + }, + "mimic": { + "path": "./demo-tasks/mimic.js", + "arguments": ["ngram-length"], + "input": { + "format": "rendered-text" + }, + "environment": { + "MY_VARIABLE": "value" + }, + "output": { + "format": "text", + "tiddler": { + "title": "HelloThere", + "type": "text/plain" + } + }, + "timeout": 100 + } + } +} +``` + +!!! Task Configuration Properties + +TBD + +!!! Input Formats + +The available input formats are: + +* `raw-text` - concatenated raw text of the tiddlers +* `rendered-text` - concatenated rendered text of the tiddlers +* `rendered-html` - concatenated rendered html of the tiddlers +* `json-raw-tiddlers` - raw tiddlers in JSON +* `json-rendered-text-tiddlers` - rendered tiddlers in JSON + +!!! Output Formats + +The available output formats include: + +* `raw-text` - raw text +* `json-raw-tiddlers` - raw tiddlers in JSON + +!! Usage + +External tasks can be invoked via the `--exec` command, or via a widget message propagated from the browser to the server. + +!!! `--exec` Command + +The `--exec` command triggers the execution of an external task. The parameters are: + +``` +tiddlywiki --exec ... +``` + +The ''taskname'' identifies the task to be run, and must match the name defined in the `tiddlywiki.info` file. + +The ''filter'' identifies the tiddlers to be passed to the external task + +The remaining ''arguments'' are passed through to the external task. + +!!! `tm-exec-task` Message + +TBD + +!! Examples + +The edition `editions/externaltasksdemo` in the TiddlyWiki5 repo contains a demo wiki that invokes sample tasks. + +!!! Mimic + +The `mimic.js` example is a simple command line tool that accepts source text via stdin and outputs a sort of statistical parody of the input text. It is written without any knowledge of TiddlyWiki, and so serves as an example of using an existing text-based tool with TiddlyWiki. + +``` +tiddlywiki editions/externaltasksdemo/ --exec mimic '[[Alice in Wonderland]]' 5 4000 --build index +``` + +View the resulting wiki at `editions/externaltaskdemo/output/index.html`: the tiddler "HelloThere" will contain a garbled version of the "Alice in Wonderland" text. + +!!! Stats + +The `stats.js` example is a simple command line tool that accepts source text via stdin and outputs a sort of statistical parody of the input text. It is written without any knowledge of TiddlyWiki, and so serves as an example of using an existing text-based tool with TiddlyWiki. + +``` +tiddlywiki editions/externaltasksdemo/ --exec stats '[[Alice in Wonderland]]' --build index +``` + +View the resulting wiki at `editions/externaltaskdemo/output/index.html`: the tiddler "HelloThere" will contain statistics about the "Alice in Wonderland" text. diff --git a/editions/externaltasksdemo/demo-tasks/mimic.js b/editions/externaltasksdemo/demo-tasks/mimic.js new file mode 100755 index 000000000..f4ed4ddcd --- /dev/null +++ b/editions/externaltasksdemo/demo-tasks/mimic.js @@ -0,0 +1,106 @@ +#!/usr/bin/env node + +/* +Reads source text from stdin and mimics it to stdout to stdout using a simple statistical analysis of ngram frequency + + mimic.js + +This utility is provided as an example of using an external task that doesn't have any prior knowledge of +TiddlyWiki. Like many Unix utilities, it just reads input from stdin and writes its output to stdout. + +*/ + +var paramNgramLength = parseInt(process.argv[2] || "",10) || 3, // Size of ngrams for mimicing + paramOutputLength = parseInt(process.argv[3] || "",10) || 1000; + +process.stdin.resume(); +process.stdin.setEncoding("utf8"); +var inputChunks = []; +process.stdin.on("data",function(chunk) { + inputChunks.push(chunk); +}); +process.stdin.on("end",function() { + // Do the mimicry + var output = mimic(inputChunks.join(""),paramNgramLength); + // Output the result + process.stdout.write(output); +}); + +function mimic(sourceText,paramNgramLength) { + var tree = {}; + scanText(tree,sourceText,paramNgramLength); + return generateText(tree,sourceText,paramNgramLength,paramOutputLength); +} + +/* +The source text is scanned to build a tree of the ngram prefixes as follows: + +{ + "abc": { // The ngram prefix described by this record + count: 42, // The number of times the prefix is found in the source text + next: [ // An array of information about each subsequent character encountered after the prefix + {char: "d", count: 41}, + {char: " ", count: 1} + ] + }, + "def": ... etc +} + +*/ + +// Process the source text into the specified tree with the chosen ngram size +function scanText(tree,sourceText,size) { + var currgram = [],ptr,c,ngram,branch,n; + if(sourceText.length <= size*2) + return tree; + sourceText += sourceText.substring(0,size*2-1); // Wrap the text around + for(ptr=0; ptr= branch.next[n].count) { + r = r - branch.next[n].count; + n++; + } + c = branch.next[n].char; + result.push(c); + currgram.push(c) + currgram.shift(); + } + return result.join(""); +} \ No newline at end of file diff --git a/editions/externaltasksdemo/demo-tasks/readme.md b/editions/externaltasksdemo/demo-tasks/readme.md new file mode 100644 index 000000000..f236e6bc2 --- /dev/null +++ b/editions/externaltasksdemo/demo-tasks/readme.md @@ -0,0 +1,4 @@ +# Sample External Tasks for TiddlyWiki + +"External Tasks" are external shell commands that can be triggered from within TiddlyWiki to perform arbitrary processing on a group of tiddlers. + diff --git a/editions/externaltasksdemo/demo-tasks/stats.js b/editions/externaltasksdemo/demo-tasks/stats.js new file mode 100755 index 000000000..aecb5be6a --- /dev/null +++ b/editions/externaltasksdemo/demo-tasks/stats.js @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +/* +Reads JSON tiddlers from stdin and outputs stats to stdout + + stats.js + +This utility is provided as an example of an external task that understands tiddler objects encoded in JSON. + +It expects to read an array of tiddler objects from stdin in this format: + + [ + { + "title": "Tiddler Title", + "text": "Text of tiddler", + "tags": "MyTag [[My Other Tag]]" + }, + ... + ] + +The output is in the same format. + +*/ + +process.stdin.resume(); +process.stdin.setEncoding("utf8"); +var inputChunks = []; +process.stdin.on("data",function(chunk) { + inputChunks.push(chunk); +}); +process.stdin.on("end",function() { + // Read the JSON input + var json = inputChunks.join(""), + data; + try { + data = JSON.parse(json); + } catch(e) { + throw "Malformed JSON: " + e + "\n\n" + json; + } + // Compute some stats + var output = computeStats(data); + // Output the result + process.stdout.write(JSON.stringify(output)); +}); + +function computeStats(tiddlers) { + var numTiddlers = tiddlers.length, + wordCount = 0, + wordFrequency = {}; + tiddlers.forEach(function(tiddler) { + var matches = (tiddler.text || "").match(/[A-Za-z0-9\u00c0-\u00d6\u00d8-\u00de\u00df-\u00f6\u00f8-\u00ff\u0150\u0170\u0151\u0171]+/g); + if(matches) { + wordCount += matches.length; + matches.forEach(function(word) { + word = word.toLowerCase(); + wordFrequency[word] = wordFrequency[word] || 0; + wordFrequency[word] += 1; + }); + } + }); + var sortedWords = Object.keys(wordFrequency).sort(function(a,b) { + if(wordFrequency[a] > wordFrequency[b]) { + return -1; + } else if(wordFrequency[a] < wordFrequency[b]) { + return +1; + } else { + return 0; + } + }); + // Output + return [ + { + title: "HelloThere", + text: numTiddlers + " tiddlers in sample.\n" + wordCount + " words in sample.\n" + sortedWords.filter(function(word) { + return word.length > 1 && wordFrequency[word] > 1; + }).map(function(word) { + return word + " " + wordFrequency[word] + "\n"; + }).join(""), + type: "text/plain" + } + ]; +}; diff --git a/editions/externaltasksdemo/tiddlers/system/DefaultTiddlers.tid b/editions/externaltasksdemo/tiddlers/system/DefaultTiddlers.tid new file mode 100644 index 000000000..a3c362aff --- /dev/null +++ b/editions/externaltasksdemo/tiddlers/system/DefaultTiddlers.tid @@ -0,0 +1,3 @@ +title: $:/DefaultTiddlers + +HelloThere \ No newline at end of file diff --git a/editions/externaltasksdemo/tiddlywiki.info b/editions/externaltasksdemo/tiddlywiki.info new file mode 100644 index 000000000..7e5215b28 --- /dev/null +++ b/editions/externaltasksdemo/tiddlywiki.info @@ -0,0 +1,45 @@ +{ + "description": "Edition demonstrating external tasks", + "plugins": [ + ], + "themes": [ + "tiddlywiki/vanilla", + "tiddlywiki/snowwhite" + ], + "includeWikis": [ + "../tw5.com" + ], + "external-tasks": { + "stats": { + "path": "./demo-tasks/stats.js", + "input": { + "format": "json-raw-tiddlers" + }, + "environment": { + "MY_VARIABLE": "value" + }, + "output": { + "format": "json-raw-tiddlers" + }, + "timeout": 100 + }, + "mimic": { + "path": "./demo-tasks/mimic.js", + "arguments": ["ngram-length"], + "input": { + "format": "rendered-text" + }, + "environment": { + "MY_VARIABLE": "value" + }, + "output": { + "format": "text", + "tiddler": { + "title": "HelloThere", + "type": "text/plain" + } + }, + "timeout": 100 + } + } +}