AWS Plugin: Add s3-load and s3-rendertiddlers commands

With these commands we can do static rendering straight to an S3 bucket

plugins/tiddlywiki/aws/docs/help.tid

@@ -9,6 +9,20 @@ Perform operation on Amazon Web Services
 --aws <sub-command> [<parameter> ...]
 ```
 
+! "s3-load" subcommand
+
+Load tiddlers from files in an S3 bucket.
+
+```
+--aws s3-load <region> <bucket> <filename>...
+```
+
+* ''region'': AWS region
+* ''bucket'': name of the bucket containing the files
+* ''filename'': one or more filenames of the files to load
+
+The content in the files is deserialized according to the content type reported by S3.
+
 ! "s3-savetiddler" subcommand
 
 Save a raw tiddler to a file in an S3 bucket.
@@ -18,6 +32,7 @@ Save a raw tiddler to a file in an S3 bucket.
 ```
 
 * ''title'': title of the tiddler to save
+* ''region'': AWS region
 * ''bucket'': name of the bucket to save the saved file
 * ''filename'': filename of the saved file
 * ''zipfilename'': optional; the file will be packed into a ZIP file with the specified name
@@ -31,9 +46,27 @@ Save the results of rendering a tiddler to a file in an S3 bucket.
 ```
 
 * ''title'': title of the tiddler to render
+* ''region'': AWS region
 * ''bucket'': name of the bucket to save the rendered file
 * ''filename'': filename of the rendered file
 * ''type'': optional render type (defaults to "text/html")
 * ''template'': optional template (defaults to directly rendering the tiddler without using a template)
 * ''zipfilename'': optional; the file will be packed into a ZIP file with the specified name
 * ''savetype'': optional; the MIME type for the saved file (defaults to ''type'' or "text/html")
+
+! "s3-rendertiddlers" subcommand
+
+Save the results of rendering tiddlers identified by a filter to files in an S3 bucket.
+
+```
+--aws s3-rendertiddler <filter> <template> <region> <bucket> <prefix> <type> <extension> <savetype>
+```
+
+* ''filter'': filter identifying tiddlers to render
+* ''template'': template for rendering each tiddler
+* ''region'': AWS region
+* ''bucket'': name of the bucket to save the rendered file
+* ''prefix'': prefix for rendered file names
+* ''type'': optional render type (defaults to "text/html")
+* ''extension'': optional file extension (defaults to ".html")
+* ''savetype'': optional; the MIME type for the saved file (defaults to ''type'' or "text/html")

plugins/tiddlywiki/aws/modules/command.js

@@ -42,6 +42,39 @@ Command.prototype.execute = function() {
 
 Command.prototype.subCommands = {};
 
+// Load tiddlers from files in an S3 bucket
+Command.prototype.subCommands["s3-load"] = function() {
+	var self = this,
+		wiki = this.commander.wiki,
+		region = this.params[1],
+		bucket = this.params[2],
+		filepaths = this.params.slice(3);
+	// Check parameters
+	if(!region || !bucket || filepaths.length < 1) {
+		throw "Missing parameters";
+	}
+	async.eachLimit(
+		filepaths,
+		4,
+		function(filepath,callback) {
+			awsUtils.getTextFile(region,bucket,filepath,function(err,data) {
+				if(err) {
+					return callback(err);
+				}
+				var tiddlers = self.commander.wiki.deserializeTiddlers(data.type,data.body,{});
+				$tw.utils.each(tiddlers,function(tiddler) {
+					self.commander.wiki.importTiddler(new $tw.Tiddler(tiddler));
+				});
+				callback(null);
+			});
+		},
+		function(err,results) {
+			self.callback(err,results);
+		}
+	);
+	return null;
+};
+
 // Render a tiddler to an S3 bucket
 Command.prototype.subCommands["s3-rendertiddler"] = function() {
 	var self = this,
@@ -55,6 +88,10 @@ Command.prototype.subCommands["s3-rendertiddler"] = function() {
 		zipfilename = this.params[7],
 		saveType = this.params[8] || type,
 		variables = {};
+	// Check parameters
+	if(!title || !region || !bucket || !filename) {
+		throw "Missing parameters";
+	}
 	// Process the template if present
 	if(template) {
 		variables.currentTiddler = title;
@@ -75,7 +112,7 @@ Command.prototype.subCommands["s3-rendertiddler"] = function() {
 	}
 	// Save the file
 	async.series([
-		awsUtils.putFile.bind(null,region,bucket,filename,text,saveType)
+		awsUtils.putTextFile.bind(null,region,bucket,filename,text,saveType)
 	],
 	function(err,results){
 		self.callback(err,results);
@@ -83,6 +120,40 @@ Command.prototype.subCommands["s3-rendertiddler"] = function() {
 	return null;
 };
 
+Command.prototype.subCommands["s3-rendertiddlers"] = function() {
+	var self = this,
+		wiki = this.commander.wiki,
+		filter = this.params[1],
+		template = this.params[2],
+		region = this.params[3],
+		bucket = this.params[4],
+		prefix = this.params[5],
+		type = this.params[6] || "text/html",
+		extension = this.params[7] || ".html",
+		saveType = this.params[8] || type,
+		tiddlers = wiki.filterTiddlers(filter);
+	// Check parameters
+	if(!filter || !template || !region || !bucket || !prefix) {
+		throw "Missing parameters";
+	}
+	async.eachLimit(
+		tiddlers,
+		2,
+		function(title,callback) {
+			var parser = wiki.parseTiddler(template),
+				widgetNode = wiki.makeWidget(parser,{variables: {currentTiddler: title}}),
+				container = $tw.fakeDocument.createElement("div");
+			widgetNode.render(container,null);
+			var text = type === "text/html" ? container.innerHTML : container.textContent;
+			awsUtils.putTextFile(region,bucket,prefix + encodeURIComponent(title) + extension,text,saveType,callback);
+		},
+		function(err,results) {
+			self.callback(err,results);
+		}
+	);
+	return null;
+};
+
 // Save a tiddler to an S3 bucket
 Command.prototype.subCommands["s3-savetiddler"] = function() {
 	var self = this,
@@ -96,6 +167,10 @@ Command.prototype.subCommands["s3-savetiddler"] = function() {
 		text = tiddler.fields.text,
 		type = tiddler.fields.type,
 		encoding = ($tw.config.contentTypeInfo[type] || {encoding: "utf8"}).encoding;
+	// Check parameters
+	if(!title || !region || !bucket || !filename) {
+		throw "Missing parameters";
+	}
 	// Zip it if needed
 	if(zipfilename) {
 		var JSZip = require("$:/plugins/tiddlywiki/jszip/jszip.js"),
@@ -107,7 +182,7 @@ Command.prototype.subCommands["s3-savetiddler"] = function() {
 	}
 	// Save the file
 	async.series([
-		awsUtils.putFile.bind(null,region,bucket,filename,text,type)
+		awsUtils.putTextFile.bind(null,region,bucket,filename,text,type)
 	],
 	function(err,results){
 		self.callback(err,results);

plugins/tiddlywiki/aws/modules/utils.js

@@ -13,14 +13,44 @@ AWS utility functions
 "use strict";
 
 /*
-Put a file to an S3 bucket
+Get a file from an S3 bucket
+region:
+bucketName:
+title:
+callback: invoked with (err,{body:,type:}
 */
-function putFile(region,bucketName,title,text,type,callback) {
+function getTextFile(region,bucketName,title,callback) {
-console.log("Writing file",bucketName,title,type)
+console.log("Reading file from S3",bucketName,title)
 	var AWS = require("aws-sdk"),
 		s3bucket = new AWS.S3({
-		region: region
+			region: region
-	}),
+		}),
+		params = {
+			Bucket: bucketName,
+			Key: title
+		};
+	s3bucket.getObject(params,function(err,data) {
+		if(err) {
+			return callback(err);
+		}
+		callback(null,{
+			etag: data.ETag,
+			version: data.VersionId,
+			type: data.ContentType,
+			body: data.Body.toString()
+		});
+	});
+}
+
+/*
+Put a file to an S3 bucket
+*/
+function putTextFile(region,bucketName,title,text,type,callback) {
+console.log("Writing file to S3",bucketName,title,type)
+	var AWS = require("aws-sdk"),
+		s3bucket = new AWS.S3({
+			region: region
+		}),
 		encoding = ($tw.config.contentTypeInfo[type] || {encoding: "utf8"}).encoding,
 		params = {
 			Bucket: bucketName,
@@ -31,6 +61,7 @@ console.log("Writing file",bucketName,title,type)
 	s3bucket.upload(params,callback);
 }
 
-exports.putFile = putFile;
+exports.putTextFile = putTextFile;
+exports.getTextFile = getTextFile;
 
 })();