Refactor multipart form handling for more reusability

plugins/tiddlywiki/multiwikiserver/modules/init.js

@@ -30,14 +30,9 @@ exports.startup = function() {
 			databasePath: path.resolve($tw.boot.wikiPath,"store/database.sqlite"),
 			engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better"), // better || wasm
 			attachmentStore: attachmentStore
-		}),
-		MultipartFormManager = require("$:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js").MultipartFormManager,
-		multipartFormManager = new MultipartFormManager({
-			store: store
 		});
 	$tw.mws = {
-		store: store,
+		store: store
-		multipartFormManager: multipartFormManager
 	};
 	// Performance timing
 	console.time("mws-initial-load");

plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js

@@ -1,92 +0,0 @@
-/*\
-title: $:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js
-type: application/javascript
-module-type: library
-
-A class that handles an incoming multipart/form-data stream, streaming the data to temporary files
-in the store/inbox folder. It invokes a callback when all the data is available. The callback can explicitly
-claim some or all of the files, otherwise they are deleted on return from the callback. Claimed files should
-be moved out of the store/inbox folder.
-
-\*/
-
-(function() {
-
-/*
-Create an instance of the upload manager. Options include:
-
-store - sqlTiddlerStore to use for saving tiddlers
-*/
-function MultipartFormManager(options) {
-	options = options || {};
-	this.store = options.store;
-}
-
-/*
-Process a new multipart/form-data stream. Options include:
-
-state - provided by server.js
-recipe - optional name of recipe to write to (one of recipe or bag must be specified)
-bag - optional name of bag to write to (one of recipe or bag must be specified)
-callback - invoked as callback(err,results). Results is an array of {title:,bag_name:}
-
-formData is:
-{
-	parts: [
-		{
-			name: "fieldname",
-			filename: "filename",
-			filePath: "/users/home/mywiki/store/inbox/09cabc74-8163-4ead-a35b-4ca768f02d62/64131628-cbff-4677-b146-d85c42c232dc",
-			headers: {
-				name: "value",
-				...
-			}
-		},
-		...
-	]
-}
-*/
-MultipartFormManager.prototype.processNewStream = function(options) {
-	let fileStream = null;
-	let fieldValue = "";
-	state.streamMultipartData({
-		cbPartStart: function(headers,name,filename) {
-			console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
-			if(filename) {
-				fileStream = fs.createWriteStream(filename);
-			} else {
-				fieldValue = "";
-			}
-		},
-		cbPartChunk: function(chunk) {
-			if(fileStream) {
-				fileStream.write(chunk);
-			} else {
-				fieldValue = fieldValue + chunk;
-			}
-		},
-		cbPartEnd: function() {
-			if(fileStream) {
-				fileStream.end();
-				fileStream = null;
-			} else {
-				console.log("Data was " + fieldValue);
-				fieldValue = "";
-			}
-		},
-		cbFinished: function(err) {
-			if(err) {
-				state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
-			} else {
-				state.sendResponse(200, {"Content-Type": "text/plain"},"Multipart data processed");
-			}
-		}
-	});
-}
-
-MultipartFormManager.prototype.close = function() {
-};
-
-exports.MultipartFormManager = MultipartFormManager;
-
-})();

plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js

@@ -25,111 +25,46 @@ exports.csrfDisable = true;
 
 exports.handler = function(request,response,state) {
 	const path = require("path"),
-		fs = require("fs");
+		fs = require("fs"),
+		processIncomingStream = require("$:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js").processIncomingStream;
 	// Get the  parameters
 	var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
 		bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]);
 console.log(`Got ${bag_name} and ${bag_name_2}`)
-	// Require the recipe names to match
+	// Require the bag names to match
 	if(bag_name !== bag_name_2) {
 		return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: bag names do not match");
 	}
 	// Process the incoming data
-	const inboxName = $tw.utils.stringifyDate(new Date());
+	processIncomingStream({
-	const inboxPath = path.resolve($tw.mws.store.attachmentStore.storePath,"inbox",inboxName);
+		store: $tw.mws.store,
-	$tw.utils.createDirectory(inboxPath);
+		state: state,
-	let fileStream = null; // Current file being written
+		response: response,
-	let hash = null; // Accumulating hash of current part
+		bagname: bag_name,
-	let length = 0; // Accumulating length of current part
+		callback: function(err,results) {
-	const parts = [];
+			response.writeHead(200, "OK",{
-	state.streamMultipartData({
+				"Content-Type":  "text/html"
-		cbPartStart: function(headers,name,filename) {
+			});
-			console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
+			response.write(`
-			const part = {
+				<!doctype html>
-				name: name,
+				<head>
-				filename: filename,
+					<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
-				headers: headers
+				</head>
-			};
+				<body>
-			if(filename) {
+			`);
-				const inboxFilename = (parts.length).toString();
+			// Render the html
-				part.inboxFilename = path.resolve(inboxPath,inboxFilename);
+			var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{
-				fileStream = fs.createWriteStream(part.inboxFilename);
+				variables: {
-			} else {
+					"bag-name": bag_name,
-				part.value = "";
+					"imported-titles": JSON.stringify(results)
-			}
-			hash = new $tw.sjcl.hash.sha256();
-			length = 0;
-			parts.push(part)
-		},
-		cbPartChunk: function(chunk) {
-			if(fileStream) {
-				fileStream.write(chunk);
-			} else {
-				parts[parts.length - 1].value += chunk;
-			}
-			length = length + chunk.length;
-			hash.update(chunk);
-			console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`);
-		},
-		cbPartEnd: function() {
-			if(fileStream) {
-				fileStream.end();
-			}
-			fileStream = null;
-			parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString();
-			hash = null;
-		},
-		cbFinished: function(err) {
-			if(err) {
-				state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
-			} else {
-				console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`);
-				const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename);
-				if(!partFile) {
-					return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload");
 				}
-				const type = partFile.headers["content-type"];
+			});
-				const tiddlerFields = {
+			response.write(html);
-					title: partFile.filename,
+			response.write(`
-					type: type
+				</body>
-				};
+				</html>
-				for(const part of parts) {
+			`);
-					const tiddlerFieldPrefix = "tiddler-field-";
+			response.end();
-					if(part.name.startsWith(tiddlerFieldPrefix)) {
-						tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim();
-					}
-				}
-				console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`)
-				$tw.mws.store.saveBagTiddlerWithAttachment(tiddlerFields,bag_name,{
-					filepath: partFile.inboxFilename,
-					type: type,
-					hash: partFile.hash
-				});
-				$tw.utils.deleteDirectory(inboxPath);
-				response.writeHead(200, "OK",{
-					"Content-Type":  "text/html"
-				});
-				response.write(`
-					<!doctype html>
-					<head>
-						<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
-					</head>
-					<body>
-				`);
-				// Render the html
-				var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{
-					variables: {
-						"bag-name": bag_name,
-						"imported-titles": JSON.stringify([tiddlerFields.title])
-					}
-				});
-				response.write(html);
-				response.write(`
-					</body>
-					</html>
-				`);
-				response.end();
-			}
 		}
 	});
 };

plugins/tiddlywiki/multiwikiserver/modules/routes/helpers/multipart-forms.js

@@ -0,0 +1,104 @@
+/*\
+title: $:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js
+type: application/javascript
+module-type: library
+
+A function that handles an incoming multipart/form-data stream, streaming the data to temporary files
+in the store/inbox folder. Once the data is received, it imports any tiddlers and invokes a callback.
+
+\*/
+
+(function() {
+
+/*
+Process an incoming new multipart/form-data stream. Options include:
+
+store - tiddler store
+state - provided by server.js
+response - provided by server.js
+bagname - name of bag to write to
+callback - invoked as callback(err,results). Results is an array of titles of imported tiddlers
+*/
+exports.processIncomingStream = function(options) {
+	const self = this;
+	const path = require("path"),
+		fs = require("fs");
+	// Process the incoming data
+	const inboxName = $tw.utils.stringifyDate(new Date());
+	const inboxPath = path.resolve(options.store.attachmentStore.storePath,"inbox",inboxName);
+	$tw.utils.createDirectory(inboxPath);
+	let fileStream = null; // Current file being written
+	let hash = null; // Accumulating hash of current part
+	let length = 0; // Accumulating length of current part
+	const parts = []; // Array of {name:, headers:, value:, hash:} and/or {name:, filename:, headers:, inboxFilename:, hash:} 
+	options.state.streamMultipartData({
+		cbPartStart: function(headers,name,filename) {
+			console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
+			const part = {
+				name: name,
+				filename: filename,
+				headers: headers
+			};
+			if(filename) {
+				const inboxFilename = (parts.length).toString();
+				part.inboxFilename = path.resolve(inboxPath,inboxFilename);
+				fileStream = fs.createWriteStream(part.inboxFilename);
+			} else {
+				part.value = "";
+			}
+			hash = new $tw.sjcl.hash.sha256();
+			length = 0;
+			parts.push(part)
+		},
+		cbPartChunk: function(chunk) {
+			if(fileStream) {
+				fileStream.write(chunk);
+			} else {
+				parts[parts.length - 1].value += chunk;
+			}
+			length = length + chunk.length;
+			hash.update(chunk);
+			console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`);
+		},
+		cbPartEnd: function() {
+			if(fileStream) {
+				fileStream.end();
+			}
+			fileStream = null;
+			parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString();
+			hash = null;
+		},
+		cbFinished: function(err) {
+			if(err) {
+				return options.callback(err);
+			} else {
+				console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`);
+				const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename);
+				if(!partFile) {
+					return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload");
+				}
+				const type = partFile.headers["content-type"];
+				const tiddlerFields = {
+					title: partFile.filename,
+					type: type
+				};
+				for(const part of parts) {
+					const tiddlerFieldPrefix = "tiddler-field-";
+					if(part.name.startsWith(tiddlerFieldPrefix)) {
+						tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim();
+					}
+				}
+				console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`)
+				options.store.saveBagTiddlerWithAttachment(tiddlerFields,options.bagname,{
+					filepath: partFile.inboxFilename,
+					type: type,
+					hash: partFile.hash
+				});
+				$tw.utils.deleteDirectory(inboxPath);
+				options.callback(null,[tiddlerFields.title]);
+			}
+		}
+	});
+};
+
+})();