Initial commit

core/modules/commands/exec.js

@@ -0,0 +1,121 @@
+/*\
+title: $:/core/modules/commands/exec.js
+type: application/javascript
+module-type: command
+
+Command to execute an external task
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.info = {
+	name: "exec",
+	synchronous: false
+};
+
+var Command = function(params,commander,callback) {
+	this.params = params;
+	this.commander = commander;
+	this.callback = callback;
+};
+
+Command.prototype.execute = function() {
+	var self = this;
+	if(this.params.length < 2) {
+		return "Missing parameters";
+	}
+	var name = self.params[0], // External task name
+		filter = self.params[1], // Filter of tiddlers to pass to the task
+		args = self.params.slice(2); // Remaining arguments are passed to the task arguments
+	// Find the task information
+	var taskInfo = ($tw.boot.wikiInfo["external-tasks"] || {})[name];
+	if(!taskInfo) {
+		return this.callback("External task \"" + name + "\" not found");
+	}
+	// Execute it
+	var spawn = require("child_process").spawn,
+		path = require("path"),
+		childProcess = spawn(path.resolve($tw.boot.wikiPath,taskInfo.path),args,{
+			stdio: ["pipe","pipe",process.stderr],
+			shell: true,
+			env: $tw.utils.extend({},process.env,taskInfo.environment)
+		});
+	// Choose the input representation
+	var taskInfoInput = taskInfo.input || {},
+		data;
+	switch(taskInfoInput.format || "json-raw-tiddlers") {
+		case "rendered-text":
+			var titles = self.commander.wiki.filterTiddlers(filter),
+				output = [];
+			$tw.utils.each(titles,function(title) {
+				output.push(self.commander.wiki.renderTiddler("text/plain",title));
+			});
+			data = output.join(""); 
+			break;
+		case "json-rendered-text-tiddlers":
+			var titles = self.commander.wiki.filterTiddlers(filter),
+				tiddlers = [];
+			$tw.utils.each(titles,function(title) {
+				tiddlers.push({
+					title: title,
+					text: self.commander.wiki.renderTiddler("text/plain",title)
+				})
+			});
+			data = JSON.stringify(tiddlers); 
+			break;
+		case "json-raw-tiddlers":
+			// Intentional fall-through
+		default:
+			data = this.commander.wiki.getTiddlersAsJson(filter);
+			break;
+	}
+	if(data === undefined) {
+		return this.callback("No input data defined for tiddler processor");
+	}
+	// Pass the tiddlers as JSON over stdin
+	childProcess.stdin.write(data);
+	childProcess.stdin.end();
+	// Catch the output
+	var chunks = [];
+	childProcess.stdout.on("data",function(chunk) {
+		chunks.push(chunk.toString());
+	});
+	// Pick up the output when the process ends
+	childProcess.once("exit",function(code) {
+		if(code !== 0) {
+			return self.callback("Error executing external task: " + code);
+		}
+		var childOutput = chunks.join(""),
+			taskInfoOutput = taskInfo.output || {},
+			data;
+		switch(taskInfoOutput.format || "text") {
+			case "json-raw-tiddlers":
+				try {
+					data = JSON.parse(childOutput);
+				} catch(e) {
+					self.callback("Error parsing returned JSON: " + e + "\n\n\n->\n" + childOutput);
+				}
+				// Add the tiddlers
+				self.commander.wiki.addTiddlers(data);
+				break;
+			case "text":
+				// Intentional fall-through
+			default:
+				self.commander.wiki.addTiddler(new $tw.Tiddler(taskInfoOutput.tiddler,{
+					text: childOutput
+				}));
+				break;
+		}
+		// Exit successfully
+		self.callback(null);
+	});
+	return null;
+};
+
+exports.Command = Command;
+
+})();

editions/dev/tiddlers/HelloThere.tid

@@ -1,6 +1,8 @@
-modified: 20141122200310516
+created: 20181001170605910
+modified: 20181001171639702
 tags: TableOfContents
 title: HelloThere
+type: text/vnd.tiddlywiki
 
 Welcome to the developer documentation for TiddlyWiki (https://tiddlywiki.com/). It is currently a work in progress as material from two different sources is adapted and merged in addition to original content being added:
 
@@ -22,5 +24,6 @@ Welcome to the developer documentation for TiddlyWiki (https://tiddlywiki.com/).
 ** SyncAdaptorModules
 ** WidgetModules
 ** WikiRuleModules
-*Original developer documentation
+* New developer documentation
 ** HookMechanism
+** [[External Shell Tasks]]

editions/dev/tiddlers/new/External Shell Tasks.tid

@@ -0,0 +1,131 @@
+created: 20181001171604072
+modified: 20181001184306738
+title: External Shell Tasks
+type: text/vnd.tiddlywiki
+
+!! Introduction
+
+It can be difficult for experienced developers to become proficient in working with TiddlyWiki's code: besides requiring a good understanding of JavaScript, the internals are built around relatively unusual concepts and techniques that are likely to be unfamiliar to most.
+
+However, many software developers are very familiar with using and creating text-based command line tools that adhere to the Unix philosophy of using stdin/stdout for input and output, allowing tools to be chained together in powerful ways.
+
+The external task mechanism allows Unix-style tasks to be registered and executed by TiddlyWiki under Node.js. Execution can be triggered via a server command or via a message propagated from the browser.
+
+!! Configuration
+
+External tasks must be registered in the `tiddlywiki.info` file of a wiki folder. For example:
+
+```
+{
+	"description": "Edition demonstrating external tasks",
+	"plugins": [
+	],
+	"themes": [
+		"tiddlywiki/vanilla",
+		"tiddlywiki/snowwhite"
+	],
+	"includeWikis": [
+		"../tw5.com"
+	],
+	"external-tasks": {
+		"stats": {
+			"path": "./demo-tasks/stats.js",
+			"input": {
+				"format": "json-raw-tiddlers"
+			},
+			"environment": {
+				"MY_VARIABLE": "value"
+			},
+			"output": {
+				"format": "json-raw-tiddlers"
+			},
+			"timeout": 100
+		},
+		"mimic": {
+			"path": "./demo-tasks/mimic.js",
+			"arguments": ["ngram-length"],
+			"input": {
+				"format": "rendered-text"
+			},
+			"environment": {
+				"MY_VARIABLE": "value"
+			},
+			"output": {
+				"format": "text",
+				"tiddler": {
+					"title": "HelloThere",
+					"type": "text/plain"
+				}
+			},
+			"timeout": 100
+		}
+	}
+}
+```
+
+!!! Task Configuration Properties
+
+TBD
+
+!!! Input Formats
+
+The available input formats are:
+
+* `raw-text` - concatenated raw text of the tiddlers
+* `rendered-text` - concatenated rendered text of the tiddlers
+* `rendered-html` - concatenated rendered html of the tiddlers
+* `json-raw-tiddlers` - raw tiddlers in JSON
+* `json-rendered-text-tiddlers` - rendered tiddlers in JSON
+
+!!! Output Formats
+
+The available output formats include:
+
+* `raw-text` - raw text
+* `json-raw-tiddlers` - raw tiddlers in JSON
+
+!! Usage
+
+External tasks can be invoked via the `--exec` command, or via a widget message propagated from the browser to the server.
+
+!!! `--exec` Command
+
+The `--exec` command triggers the execution of an external task. The parameters are:
+
+```
+tiddlywiki <wikifolder> --exec <taskname> <filter> <arguments>...
+```
+
+The ''taskname'' identifies the task to be run, and must match the name defined in the `tiddlywiki.info` file.
+
+The ''filter'' identifies the tiddlers to be passed to the external task
+
+The remaining ''arguments'' are passed through to the external task.
+
+!!! `tm-exec-task` Message
+
+TBD
+
+!! Examples
+
+The edition `editions/externaltasksdemo` in the TiddlyWiki5 repo contains a demo wiki that invokes sample tasks.
+
+!!! Mimic
+
+The `mimic.js` example is a simple command line tool that accepts source text via stdin and outputs a sort of statistical parody of the input text. It is written without any knowledge of TiddlyWiki, and so serves as an example of using an existing text-based tool with TiddlyWiki.
+
+```
+tiddlywiki editions/externaltasksdemo/ --exec mimic '[[Alice in Wonderland]]' 5 4000 --build index
+```
+
+View the resulting wiki at `editions/externaltaskdemo/output/index.html`: the tiddler "HelloThere" will contain a garbled version of the "Alice in Wonderland" text.
+
+!!! Stats
+
+The `stats.js` example is a simple command line tool that accepts source text via stdin and outputs a sort of statistical parody of the input text. It is written without any knowledge of TiddlyWiki, and so serves as an example of using an existing text-based tool with TiddlyWiki.
+
+```
+tiddlywiki editions/externaltasksdemo/ --exec stats '[[Alice in Wonderland]]' --build index
+```
+
+View the resulting wiki at `editions/externaltaskdemo/output/index.html`: the tiddler "HelloThere" will contain statistics about the "Alice in Wonderland" text.

editions/externaltasksdemo/demo-tasks/mimic.js

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+
+/*
+Reads source text from stdin and mimics it to stdout to stdout using a simple statistical analysis of ngram frequency
+
+	mimic.js <ngram-length> <output-length>
+
+This utility is provided as an example of using an external task that doesn't have any prior knowledge of
+TiddlyWiki. Like many Unix utilities, it just reads input from stdin and writes its output to stdout.
+
+*/
+
+var paramNgramLength = parseInt(process.argv[2] || "",10) || 3, // Size of ngrams for mimicing
+	paramOutputLength = parseInt(process.argv[3] || "",10) || 1000;
+
+process.stdin.resume();
+process.stdin.setEncoding("utf8");
+var inputChunks = [];
+process.stdin.on("data",function(chunk) {
+	inputChunks.push(chunk);
+});
+process.stdin.on("end",function() {
+	// Do the mimicry
+	var output = mimic(inputChunks.join(""),paramNgramLength);
+	// Output the result
+	process.stdout.write(output);
+});
+
+function mimic(sourceText,paramNgramLength) {
+	var tree = {};
+	scanText(tree,sourceText,paramNgramLength);
+	return generateText(tree,sourceText,paramNgramLength,paramOutputLength);
+}
+
+/*
+The source text is scanned to build a tree of the ngram prefixes as follows:
+
+{
+	"abc": { // The ngram prefix described by this record
+		count: 42, // The number of times the prefix is found in the source text
+		next: [ // An array of information about each subsequent character encountered after the prefix
+			{char: "d", count: 41},
+			{char: " ", count: 1}
+		]
+	},
+	"def": ... etc
+}
+
+*/
+
+// Process the source text into the specified tree with the chosen ngram size
+function scanText(tree,sourceText,size) {
+	var currgram = [],ptr,c,ngram,branch,n;
+	if(sourceText.length <= size*2)
+		return tree;
+	sourceText += sourceText.substring(0,size*2-1); // Wrap the text around
+	for(ptr=0; ptr<size; ptr++) {
+		currgram.push(sourceText.substr(ptr,1));
+	}
+	while(ptr < sourceText.length) {
+		ngram = currgram.join("");
+		c = sourceText.substr(ptr++,1);
+		branch = tree[ngram];
+		if(branch === undefined) {
+			branch = tree[ngram] = {count: 0,next: []};
+		}
+		for(n = 0; n<branch.next.length; n++) {
+			if(branch.next[n].char === c)
+				break;
+		}
+		if(branch.next[n] === undefined) {
+			branch.next[n] = {char: c, count: 1};
+		} else {
+			branch.next[n].count++;
+		}
+		branch.count++;
+		currgram.push(c)
+		currgram.shift();
+	}
+	return tree;
+}
+
+// Use the tree to generate mimicry
+function generateText(tree,sourceText,size,length) {
+	var currgram = [];
+	for(var t=0; t<size; t++) {
+		currgram.push(sourceText.substr(t,1));
+	}
+	var result = [];
+	var c,ngram,branch,r,n;
+	for(t=0; t<length; t++) {
+		ngram = currgram.join("");
+		branch = tree[ngram];
+		n = 0;
+		r = Math.floor(Math.random() * branch.count);
+		while(r >= branch.next[n].count) {
+			r = r - branch.next[n].count;
+			n++;
+		}
+		c = branch.next[n].char;
+		result.push(c);
+		currgram.push(c)
+		currgram.shift();
+	}
+	return result.join("");
+}

editions/externaltasksdemo/demo-tasks/readme.md

@@ -0,0 +1,4 @@
+# Sample External Tasks for TiddlyWiki
+
+"External Tasks" are external shell commands that can be triggered from within TiddlyWiki to perform arbitrary processing on a group of tiddlers.
+

editions/externaltasksdemo/demo-tasks/stats.js

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+
+/*
+Reads JSON tiddlers from stdin and outputs stats to stdout
+
+	stats.js
+
+This utility is provided as an example of an external task that understands tiddler objects encoded in JSON.
+
+It expects to read an array of tiddler objects from stdin in this format:
+
+	[
+		{
+			"title": "Tiddler Title",
+			"text": "Text of tiddler",
+			"tags": "MyTag [[My Other Tag]]"
+		},
+		...
+	]
+
+The output is in the same format.
+
+*/
+
+process.stdin.resume();
+process.stdin.setEncoding("utf8");
+var inputChunks = [];
+process.stdin.on("data",function(chunk) {
+	inputChunks.push(chunk);
+});
+process.stdin.on("end",function() {
+	// Read the JSON input
+	var json = inputChunks.join(""),
+		data;
+	try {
+		data = JSON.parse(json);
+	} catch(e) {
+		throw "Malformed JSON: " + e + "\n\n" + json;
+	}
+	// Compute some stats
+	var output = computeStats(data);
+	// Output the result
+	process.stdout.write(JSON.stringify(output));
+});
+
+function computeStats(tiddlers) {
+	var numTiddlers = tiddlers.length,
+		wordCount = 0,
+		wordFrequency = {};
+	tiddlers.forEach(function(tiddler) {
+	    var matches = (tiddler.text || "").match(/[A-Za-z0-9\u00c0-\u00d6\u00d8-\u00de\u00df-\u00f6\u00f8-\u00ff\u0150\u0170\u0151\u0171]+/g);
+	    if(matches) {
+	    	wordCount += matches.length;
+	    	matches.forEach(function(word) {
+	    		word = word.toLowerCase();
+	    		wordFrequency[word] = wordFrequency[word] || 0;
+	    		wordFrequency[word] += 1;
+	    	});
+	    }
+	});
+	var sortedWords = Object.keys(wordFrequency).sort(function(a,b) {
+		if(wordFrequency[a] > wordFrequency[b]) {
+			return -1;
+		} else if(wordFrequency[a] < wordFrequency[b]) {
+			return +1;
+		} else {
+			return 0;
+		}
+	});
+	// Output
+	return [
+		{
+			title: "HelloThere",
+			text: numTiddlers + " tiddlers in sample.\n" + wordCount + " words in sample.\n" + sortedWords.filter(function(word) {
+				return word.length > 1 && wordFrequency[word] > 1;
+			}).map(function(word) {
+				return word + " " + wordFrequency[word] + "\n";
+			}).join(""),
+			type: "text/plain"
+		}
+	];
+};

editions/externaltasksdemo/tiddlers/system/DefaultTiddlers.tid

@@ -0,0 +1,3 @@
+title: $:/DefaultTiddlers
+
+HelloThere

editions/externaltasksdemo/tiddlywiki.info

@@ -0,0 +1,45 @@
+{
+	"description": "Edition demonstrating external tasks",
+	"plugins": [
+	],
+	"themes": [
+		"tiddlywiki/vanilla",
+		"tiddlywiki/snowwhite"
+	],
+	"includeWikis": [
+		"../tw5.com"
+	],
+	"external-tasks": {
+		"stats": {
+			"path": "./demo-tasks/stats.js",
+			"input": {
+				"format": "json-raw-tiddlers"
+			},
+			"environment": {
+				"MY_VARIABLE": "value"
+			},
+			"output": {
+				"format": "json-raw-tiddlers"
+			},
+			"timeout": 100
+		},
+		"mimic": {
+			"path": "./demo-tasks/mimic.js",
+			"arguments": ["ngram-length"],
+			"input": {
+				"format": "rendered-text"
+			},
+			"environment": {
+				"MY_VARIABLE": "value"
+			},
+			"output": {
+				"format": "text",
+				"tiddler": {
+					"title": "HelloThere",
+					"type": "text/plain"
+				}
+			},
+			"timeout": 100
+		}
+	}
+}