Refactor multipart form handling for more reusability
This commit is contained in:
Jeremy Ruston
parent
e3b27768d2
commit
0f5dfb89ad
|
|
@ -30,14 +30,9 @@ exports.startup = function() {
|
||||||
databasePath: path.resolve($tw.boot.wikiPath,"store/database.sqlite"),
|
databasePath: path.resolve($tw.boot.wikiPath,"store/database.sqlite"),
|
||||||
engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better"), // better || wasm
|
engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better"), // better || wasm
|
||||||
attachmentStore: attachmentStore
|
attachmentStore: attachmentStore
|
||||||
}),
|
|
||||||
MultipartFormManager = require("$:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js").MultipartFormManager,
|
|
||||||
multipartFormManager = new MultipartFormManager({
|
|
||||||
store: store
|
|
||||||
});
|
});
|
||||||
$tw.mws = {
|
$tw.mws = {
|
||||||
store: store,
|
store: store
|
||||||
multipartFormManager: multipartFormManager
|
|
||||||
};
|
};
|
||||||
// Performance timing
|
// Performance timing
|
||||||
console.time("mws-initial-load");
|
console.time("mws-initial-load");
|
||||||
|
|
|
||||||
|
|
@ -1,92 +0,0 @@
|
||||||
/*\
|
|
||||||
title: $:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js
|
|
||||||
type: application/javascript
|
|
||||||
module-type: library
|
|
||||||
|
|
||||||
A class that handles an incoming multipart/form-data stream, streaming the data to temporary files
|
|
||||||
in the store/inbox folder. It invokes a callback when all the data is available. The callback can explicitly
|
|
||||||
claim some or all of the files, otherwise they are deleted on return from the callback. Claimed files should
|
|
||||||
be moved out of the store/inbox folder.
|
|
||||||
|
|
||||||
\*/
|
|
||||||
|
|
||||||
(function() {
|
|
||||||
|
|
||||||
/*
|
|
||||||
Create an instance of the upload manager. Options include:
|
|
||||||
|
|
||||||
store - sqlTiddlerStore to use for saving tiddlers
|
|
||||||
*/
|
|
||||||
function MultipartFormManager(options) {
|
|
||||||
options = options || {};
|
|
||||||
this.store = options.store;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
Process a new multipart/form-data stream. Options include:
|
|
||||||
|
|
||||||
state - provided by server.js
|
|
||||||
recipe - optional name of recipe to write to (one of recipe or bag must be specified)
|
|
||||||
bag - optional name of bag to write to (one of recipe or bag must be specified)
|
|
||||||
callback - invoked as callback(err,results). Results is an array of {title:,bag_name:}
|
|
||||||
|
|
||||||
formData is:
|
|
||||||
{
|
|
||||||
parts: [
|
|
||||||
{
|
|
||||||
name: "fieldname",
|
|
||||||
filename: "filename",
|
|
||||||
filePath: "/users/home/mywiki/store/inbox/09cabc74-8163-4ead-a35b-4ca768f02d62/64131628-cbff-4677-b146-d85c42c232dc",
|
|
||||||
headers: {
|
|
||||||
name: "value",
|
|
||||||
...
|
|
||||||
}
|
|
||||||
},
|
|
||||||
...
|
|
||||||
]
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
MultipartFormManager.prototype.processNewStream = function(options) {
|
|
||||||
let fileStream = null;
|
|
||||||
let fieldValue = "";
|
|
||||||
state.streamMultipartData({
|
|
||||||
cbPartStart: function(headers,name,filename) {
|
|
||||||
console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
|
|
||||||
if(filename) {
|
|
||||||
fileStream = fs.createWriteStream(filename);
|
|
||||||
} else {
|
|
||||||
fieldValue = "";
|
|
||||||
}
|
|
||||||
},
|
|
||||||
cbPartChunk: function(chunk) {
|
|
||||||
if(fileStream) {
|
|
||||||
fileStream.write(chunk);
|
|
||||||
} else {
|
|
||||||
fieldValue = fieldValue + chunk;
|
|
||||||
}
|
|
||||||
},
|
|
||||||
cbPartEnd: function() {
|
|
||||||
if(fileStream) {
|
|
||||||
fileStream.end();
|
|
||||||
fileStream = null;
|
|
||||||
} else {
|
|
||||||
console.log("Data was " + fieldValue);
|
|
||||||
fieldValue = "";
|
|
||||||
}
|
|
||||||
},
|
|
||||||
cbFinished: function(err) {
|
|
||||||
if(err) {
|
|
||||||
state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
|
|
||||||
} else {
|
|
||||||
state.sendResponse(200, {"Content-Type": "text/plain"},"Multipart data processed");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
MultipartFormManager.prototype.close = function() {
|
|
||||||
};
|
|
||||||
|
|
||||||
exports.MultipartFormManager = MultipartFormManager;
|
|
||||||
|
|
||||||
})();
|
|
||||||
|
|
@ -25,111 +25,46 @@ exports.csrfDisable = true;
|
||||||
|
|
||||||
exports.handler = function(request,response,state) {
|
exports.handler = function(request,response,state) {
|
||||||
const path = require("path"),
|
const path = require("path"),
|
||||||
fs = require("fs");
|
fs = require("fs"),
|
||||||
|
processIncomingStream = require("$:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js").processIncomingStream;
|
||||||
// Get the parameters
|
// Get the parameters
|
||||||
var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
|
var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
|
||||||
bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]);
|
bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]);
|
||||||
console.log(`Got ${bag_name} and ${bag_name_2}`)
|
console.log(`Got ${bag_name} and ${bag_name_2}`)
|
||||||
// Require the recipe names to match
|
// Require the bag names to match
|
||||||
if(bag_name !== bag_name_2) {
|
if(bag_name !== bag_name_2) {
|
||||||
return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: bag names do not match");
|
return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: bag names do not match");
|
||||||
}
|
}
|
||||||
// Process the incoming data
|
// Process the incoming data
|
||||||
const inboxName = $tw.utils.stringifyDate(new Date());
|
processIncomingStream({
|
||||||
const inboxPath = path.resolve($tw.mws.store.attachmentStore.storePath,"inbox",inboxName);
|
store: $tw.mws.store,
|
||||||
$tw.utils.createDirectory(inboxPath);
|
state: state,
|
||||||
let fileStream = null; // Current file being written
|
response: response,
|
||||||
let hash = null; // Accumulating hash of current part
|
bagname: bag_name,
|
||||||
let length = 0; // Accumulating length of current part
|
callback: function(err,results) {
|
||||||
const parts = [];
|
response.writeHead(200, "OK",{
|
||||||
state.streamMultipartData({
|
"Content-Type": "text/html"
|
||||||
cbPartStart: function(headers,name,filename) {
|
});
|
||||||
console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
|
response.write(`
|
||||||
const part = {
|
<!doctype html>
|
||||||
name: name,
|
<head>
|
||||||
filename: filename,
|
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
|
||||||
headers: headers
|
</head>
|
||||||
};
|
<body>
|
||||||
if(filename) {
|
`);
|
||||||
const inboxFilename = (parts.length).toString();
|
// Render the html
|
||||||
part.inboxFilename = path.resolve(inboxPath,inboxFilename);
|
var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{
|
||||||
fileStream = fs.createWriteStream(part.inboxFilename);
|
variables: {
|
||||||
} else {
|
"bag-name": bag_name,
|
||||||
part.value = "";
|
"imported-titles": JSON.stringify(results)
|
||||||
}
|
|
||||||
hash = new $tw.sjcl.hash.sha256();
|
|
||||||
length = 0;
|
|
||||||
parts.push(part)
|
|
||||||
},
|
|
||||||
cbPartChunk: function(chunk) {
|
|
||||||
if(fileStream) {
|
|
||||||
fileStream.write(chunk);
|
|
||||||
} else {
|
|
||||||
parts[parts.length - 1].value += chunk;
|
|
||||||
}
|
|
||||||
length = length + chunk.length;
|
|
||||||
hash.update(chunk);
|
|
||||||
console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`);
|
|
||||||
},
|
|
||||||
cbPartEnd: function() {
|
|
||||||
if(fileStream) {
|
|
||||||
fileStream.end();
|
|
||||||
}
|
|
||||||
fileStream = null;
|
|
||||||
parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString();
|
|
||||||
hash = null;
|
|
||||||
},
|
|
||||||
cbFinished: function(err) {
|
|
||||||
if(err) {
|
|
||||||
state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
|
|
||||||
} else {
|
|
||||||
console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`);
|
|
||||||
const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename);
|
|
||||||
if(!partFile) {
|
|
||||||
return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload");
|
|
||||||
}
|
}
|
||||||
const type = partFile.headers["content-type"];
|
});
|
||||||
const tiddlerFields = {
|
response.write(html);
|
||||||
title: partFile.filename,
|
response.write(`
|
||||||
type: type
|
</body>
|
||||||
};
|
</html>
|
||||||
for(const part of parts) {
|
`);
|
||||||
const tiddlerFieldPrefix = "tiddler-field-";
|
response.end();
|
||||||
if(part.name.startsWith(tiddlerFieldPrefix)) {
|
|
||||||
tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`)
|
|
||||||
$tw.mws.store.saveBagTiddlerWithAttachment(tiddlerFields,bag_name,{
|
|
||||||
filepath: partFile.inboxFilename,
|
|
||||||
type: type,
|
|
||||||
hash: partFile.hash
|
|
||||||
});
|
|
||||||
$tw.utils.deleteDirectory(inboxPath);
|
|
||||||
response.writeHead(200, "OK",{
|
|
||||||
"Content-Type": "text/html"
|
|
||||||
});
|
|
||||||
response.write(`
|
|
||||||
<!doctype html>
|
|
||||||
<head>
|
|
||||||
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
|
|
||||||
</head>
|
|
||||||
<body>
|
|
||||||
`);
|
|
||||||
// Render the html
|
|
||||||
var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{
|
|
||||||
variables: {
|
|
||||||
"bag-name": bag_name,
|
|
||||||
"imported-titles": JSON.stringify([tiddlerFields.title])
|
|
||||||
}
|
|
||||||
});
|
|
||||||
response.write(html);
|
|
||||||
response.write(`
|
|
||||||
</body>
|
|
||||||
</html>
|
|
||||||
`);
|
|
||||||
response.end();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
/*\
|
||||||
|
title: $:/plugins/tiddlywiki/multiwikiserver/routes/helpers/multipart-forms.js
|
||||||
|
type: application/javascript
|
||||||
|
module-type: library
|
||||||
|
|
||||||
|
A function that handles an incoming multipart/form-data stream, streaming the data to temporary files
|
||||||
|
in the store/inbox folder. Once the data is received, it imports any tiddlers and invokes a callback.
|
||||||
|
|
||||||
|
\*/
|
||||||
|
|
||||||
|
(function() {
|
||||||
|
|
||||||
|
/*
|
||||||
|
Process an incoming new multipart/form-data stream. Options include:
|
||||||
|
|
||||||
|
store - tiddler store
|
||||||
|
state - provided by server.js
|
||||||
|
response - provided by server.js
|
||||||
|
bagname - name of bag to write to
|
||||||
|
callback - invoked as callback(err,results). Results is an array of titles of imported tiddlers
|
||||||
|
*/
|
||||||
|
exports.processIncomingStream = function(options) {
|
||||||
|
const self = this;
|
||||||
|
const path = require("path"),
|
||||||
|
fs = require("fs");
|
||||||
|
// Process the incoming data
|
||||||
|
const inboxName = $tw.utils.stringifyDate(new Date());
|
||||||
|
const inboxPath = path.resolve(options.store.attachmentStore.storePath,"inbox",inboxName);
|
||||||
|
$tw.utils.createDirectory(inboxPath);
|
||||||
|
let fileStream = null; // Current file being written
|
||||||
|
let hash = null; // Accumulating hash of current part
|
||||||
|
let length = 0; // Accumulating length of current part
|
||||||
|
const parts = []; // Array of {name:, headers:, value:, hash:} and/or {name:, filename:, headers:, inboxFilename:, hash:}
|
||||||
|
options.state.streamMultipartData({
|
||||||
|
cbPartStart: function(headers,name,filename) {
|
||||||
|
console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
|
||||||
|
const part = {
|
||||||
|
name: name,
|
||||||
|
filename: filename,
|
||||||
|
headers: headers
|
||||||
|
};
|
||||||
|
if(filename) {
|
||||||
|
const inboxFilename = (parts.length).toString();
|
||||||
|
part.inboxFilename = path.resolve(inboxPath,inboxFilename);
|
||||||
|
fileStream = fs.createWriteStream(part.inboxFilename);
|
||||||
|
} else {
|
||||||
|
part.value = "";
|
||||||
|
}
|
||||||
|
hash = new $tw.sjcl.hash.sha256();
|
||||||
|
length = 0;
|
||||||
|
parts.push(part)
|
||||||
|
},
|
||||||
|
cbPartChunk: function(chunk) {
|
||||||
|
if(fileStream) {
|
||||||
|
fileStream.write(chunk);
|
||||||
|
} else {
|
||||||
|
parts[parts.length - 1].value += chunk;
|
||||||
|
}
|
||||||
|
length = length + chunk.length;
|
||||||
|
hash.update(chunk);
|
||||||
|
console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`);
|
||||||
|
},
|
||||||
|
cbPartEnd: function() {
|
||||||
|
if(fileStream) {
|
||||||
|
fileStream.end();
|
||||||
|
}
|
||||||
|
fileStream = null;
|
||||||
|
parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString();
|
||||||
|
hash = null;
|
||||||
|
},
|
||||||
|
cbFinished: function(err) {
|
||||||
|
if(err) {
|
||||||
|
return options.callback(err);
|
||||||
|
} else {
|
||||||
|
console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`);
|
||||||
|
const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename);
|
||||||
|
if(!partFile) {
|
||||||
|
return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload");
|
||||||
|
}
|
||||||
|
const type = partFile.headers["content-type"];
|
||||||
|
const tiddlerFields = {
|
||||||
|
title: partFile.filename,
|
||||||
|
type: type
|
||||||
|
};
|
||||||
|
for(const part of parts) {
|
||||||
|
const tiddlerFieldPrefix = "tiddler-field-";
|
||||||
|
if(part.name.startsWith(tiddlerFieldPrefix)) {
|
||||||
|
tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`)
|
||||||
|
options.store.saveBagTiddlerWithAttachment(tiddlerFields,options.bagname,{
|
||||||
|
filepath: partFile.inboxFilename,
|
||||||
|
type: type,
|
||||||
|
hash: partFile.hash
|
||||||
|
});
|
||||||
|
$tw.utils.deleteDirectory(inboxPath);
|
||||||
|
options.callback(null,[tiddlerFields.title]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
})();
|
||||||
Loading…
Reference in New Issue