From 0f5dfb89ad4de2b5746bd8e3b29f866a117bdd74 Mon Sep 17 00:00:00 2001 From: Jeremy Ruston Date: Sun, 10 Mar 2024 20:20:06 +0000 Subject: [PATCH] Refactor multipart form handling for more reusability --- .../multiwikiserver/modules/init.js | 7 +- .../modules/multipart-form-manager.js | 92 ------------- .../modules/route-post-bag-tiddlers.js | 127 +++++------------- .../modules/routes/helpers/multipart-forms.js | 104 ++++++++++++++ 4 files changed, 136 insertions(+), 194 deletions(-) delete mode 100644 plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js create mode 100644 plugins/tiddlywiki/multiwikiserver/modules/routes/helpers/multipart-forms.js diff --git a/plugins/tiddlywiki/multiwikiserver/modules/init.js b/plugins/tiddlywiki/multiwikiserver/modules/init.js index c3a7f2911..a4a36d333 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/init.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/init.js @@ -30,14 +30,9 @@ exports.startup = function() { databasePath: path.resolve($tw.boot.wikiPath,"store/database.sqlite"), engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better"), // better || wasm attachmentStore: attachmentStore - }), - MultipartFormManager = require("$:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js").MultipartFormManager, - multipartFormManager = new MultipartFormManager({ - store: store }); $tw.mws = { - store: store, - multipartFormManager: multipartFormManager + store: store }; // Performance timing console.time("mws-initial-load"); diff --git a/plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js b/plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js deleted file mode 100644 index 616187147..000000000 --- a/plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js +++ /dev/null @@ -1,92 +0,0 @@ -/*\ -title: $:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js -type: application/javascript -module-type: library - -A class that handles an incoming multipart/form-data stream, streaming the data to temporary files -in the store/inbox folder. It invokes a callback when all the data is available. The callback can explicitly -claim some or all of the files, otherwise they are deleted on return from the callback. Claimed files should -be moved out of the store/inbox folder. - -\*/ - -(function() { - -/* -Create an instance of the upload manager. Options include: - -store - sqlTiddlerStore to use for saving tiddlers -*/ -function MultipartFormManager(options) { - options = options || {}; - this.store = options.store; -} - -/* -Process a new multipart/form-data stream. Options include: - -state - provided by server.js -recipe - optional name of recipe to write to (one of recipe or bag must be specified) -bag - optional name of bag to write to (one of recipe or bag must be specified) -callback - invoked as callback(err,results). Results is an array of {title:,bag_name:} - -formData is: -{ - parts: [ - { - name: "fieldname", - filename: "filename", - filePath: "/users/home/mywiki/store/inbox/09cabc74-8163-4ead-a35b-4ca768f02d62/64131628-cbff-4677-b146-d85c42c232dc", - headers: { - name: "value", - ... - } - }, - ... - ] -} -*/ -MultipartFormManager.prototype.processNewStream = function(options) { - let fileStream = null; - let fieldValue = ""; - state.streamMultipartData({ - cbPartStart: function(headers,name,filename) { - console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`) - if(filename) { - fileStream = fs.createWriteStream(filename); - } else { - fieldValue = ""; - } - }, - cbPartChunk: function(chunk) { - if(fileStream) { - fileStream.write(chunk); - } else { - fieldValue = fieldValue + chunk; - } - }, - cbPartEnd: function() { - if(fileStream) { - fileStream.end(); - fileStream = null; - } else { - console.log("Data was " + fieldValue); - fieldValue = ""; - } - }, - cbFinished: function(err) { - if(err) { - state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err); - } else { - state.sendResponse(200, {"Content-Type": "text/plain"},"Multipart data processed"); - } - } - }); -} - -MultipartFormManager.prototype.close = function() { -}; - -exports.MultipartFormManager = MultipartFormManager; - -})(); \ No newline at end of file diff --git a/plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js b/plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js index a081cc400..2c4891f4e 100644 --- a/plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js +++ b/plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js @@ -25,111 +25,46 @@ exports.csrfDisable = true; exports.handler = function(request,response,state) { const path = require("path"), - fs = require("fs"); + fs = require("fs"), + processIncomingStream = require("$:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js").processIncomingStream; // Get the parameters var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]), bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]); console.log(`Got ${bag_name} and ${bag_name_2}`) - // Require the recipe names to match + // Require the bag names to match if(bag_name !== bag_name_2) { return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: bag names do not match"); } // Process the incoming data - const inboxName = $tw.utils.stringifyDate(new Date()); - const inboxPath = path.resolve($tw.mws.store.attachmentStore.storePath,"inbox",inboxName); - $tw.utils.createDirectory(inboxPath); - let fileStream = null; // Current file being written - let hash = null; // Accumulating hash of current part - let length = 0; // Accumulating length of current part - const parts = []; - state.streamMultipartData({ - cbPartStart: function(headers,name,filename) { - console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`) - const part = { - name: name, - filename: filename, - headers: headers - }; - if(filename) { - const inboxFilename = (parts.length).toString(); - part.inboxFilename = path.resolve(inboxPath,inboxFilename); - fileStream = fs.createWriteStream(part.inboxFilename); - } else { - part.value = ""; - } - hash = new $tw.sjcl.hash.sha256(); - length = 0; - parts.push(part) - }, - cbPartChunk: function(chunk) { - if(fileStream) { - fileStream.write(chunk); - } else { - parts[parts.length - 1].value += chunk; - } - length = length + chunk.length; - hash.update(chunk); - console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`); - }, - cbPartEnd: function() { - if(fileStream) { - fileStream.end(); - } - fileStream = null; - parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString(); - hash = null; - }, - cbFinished: function(err) { - if(err) { - state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err); - } else { - console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`); - const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename); - if(!partFile) { - return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload"); + processIncomingStream({ + store: $tw.mws.store, + state: state, + response: response, + bagname: bag_name, + callback: function(err,results) { + response.writeHead(200, "OK",{ + "Content-Type": "text/html" + }); + response.write(` + + + + + + `); + // Render the html + var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{ + variables: { + "bag-name": bag_name, + "imported-titles": JSON.stringify(results) } - const type = partFile.headers["content-type"]; - const tiddlerFields = { - title: partFile.filename, - type: type - }; - for(const part of parts) { - const tiddlerFieldPrefix = "tiddler-field-"; - if(part.name.startsWith(tiddlerFieldPrefix)) { - tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim(); - } - } - console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`) - $tw.mws.store.saveBagTiddlerWithAttachment(tiddlerFields,bag_name,{ - filepath: partFile.inboxFilename, - type: type, - hash: partFile.hash - }); - $tw.utils.deleteDirectory(inboxPath); - response.writeHead(200, "OK",{ - "Content-Type": "text/html" - }); - response.write(` - - - - - - `); - // Render the html - var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{ - variables: { - "bag-name": bag_name, - "imported-titles": JSON.stringify([tiddlerFields.title]) - } - }); - response.write(html); - response.write(` - - - `); - response.end(); - } + }); + response.write(html); + response.write(` + + + `); + response.end(); } }); }; diff --git a/plugins/tiddlywiki/multiwikiserver/modules/routes/helpers/multipart-forms.js b/plugins/tiddlywiki/multiwikiserver/modules/routes/helpers/multipart-forms.js new file mode 100644 index 000000000..313bda0a2 --- /dev/null +++ b/plugins/tiddlywiki/multiwikiserver/modules/routes/helpers/multipart-forms.js @@ -0,0 +1,104 @@ +/*\ +title: $:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js +type: application/javascript +module-type: library + +A function that handles an incoming multipart/form-data stream, streaming the data to temporary files +in the store/inbox folder. Once the data is received, it imports any tiddlers and invokes a callback. + +\*/ + +(function() { + +/* +Process an incoming new multipart/form-data stream. Options include: + +store - tiddler store +state - provided by server.js +response - provided by server.js +bagname - name of bag to write to +callback - invoked as callback(err,results). Results is an array of titles of imported tiddlers +*/ +exports.processIncomingStream = function(options) { + const self = this; + const path = require("path"), + fs = require("fs"); + // Process the incoming data + const inboxName = $tw.utils.stringifyDate(new Date()); + const inboxPath = path.resolve(options.store.attachmentStore.storePath,"inbox",inboxName); + $tw.utils.createDirectory(inboxPath); + let fileStream = null; // Current file being written + let hash = null; // Accumulating hash of current part + let length = 0; // Accumulating length of current part + const parts = []; // Array of {name:, headers:, value:, hash:} and/or {name:, filename:, headers:, inboxFilename:, hash:} + options.state.streamMultipartData({ + cbPartStart: function(headers,name,filename) { + console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`) + const part = { + name: name, + filename: filename, + headers: headers + }; + if(filename) { + const inboxFilename = (parts.length).toString(); + part.inboxFilename = path.resolve(inboxPath,inboxFilename); + fileStream = fs.createWriteStream(part.inboxFilename); + } else { + part.value = ""; + } + hash = new $tw.sjcl.hash.sha256(); + length = 0; + parts.push(part) + }, + cbPartChunk: function(chunk) { + if(fileStream) { + fileStream.write(chunk); + } else { + parts[parts.length - 1].value += chunk; + } + length = length + chunk.length; + hash.update(chunk); + console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`); + }, + cbPartEnd: function() { + if(fileStream) { + fileStream.end(); + } + fileStream = null; + parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString(); + hash = null; + }, + cbFinished: function(err) { + if(err) { + return options.callback(err); + } else { + console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`); + const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename); + if(!partFile) { + return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload"); + } + const type = partFile.headers["content-type"]; + const tiddlerFields = { + title: partFile.filename, + type: type + }; + for(const part of parts) { + const tiddlerFieldPrefix = "tiddler-field-"; + if(part.name.startsWith(tiddlerFieldPrefix)) { + tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim(); + } + } + console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`) + options.store.saveBagTiddlerWithAttachment(tiddlerFields,options.bagname,{ + filepath: partFile.inboxFilename, + type: type, + hash: partFile.hash + }); + $tw.utils.deleteDirectory(inboxPath); + options.callback(null,[tiddlerFields.title]); + } + } + }); +}; + +})(); \ No newline at end of file