MWS: Add support for large tiddlers to be stored as attachment files

Fixes #8022

core/modules/server/server.js

@@ -27,7 +27,6 @@ A simple HTTP server with regexp-based routes
 options: variables - optional hashmap of variables to set (a misnomer - they are really constant parameters)
 		 routes - optional array of routes to use
 		 wiki - reference to wiki object
-		 verbose - boolean
 */
 function Server(options) {
 	var self = this;
@@ -35,7 +34,6 @@ function Server(options) {
 	this.authenticators = options.authenticators || [];
 	this.wiki = options.wiki;
 	this.boot = options.boot || $tw.boot;
-	this.verbose = !!options.verbose;
 	// Initialise the variables
 	this.variables = $tw.utils.extend({},this.defaultVariables);
 	if(options.variables) {
@@ -45,14 +43,6 @@ function Server(options) {
 			}
 		}
 	}
-	// Register server extensions
-	this.extensions = [];
-	$tw.modules.forEachModuleOfType("server-extension",function(title,exports) {
-		var extension = new exports.Extension(self);
-		self.extensions.push(extension);
-	});
-	// Initialise server extensions
-	this.invokeExtensionHook("server-start-initialisation");
 	// Setup the default required plugins
 	this.requiredPlugins = this.get("required-plugins").split(',');
 	// Initialise CSRF
@@ -105,16 +95,8 @@ function Server(options) {
 	this.servername = $tw.utils.transliterateToSafeASCII(this.get("server-name") || this.wiki.getTiddlerText("$:/SiteTitle") || "TiddlyWiki5");
 	this.boot.origin = this.get("origin")? this.get("origin"): this.protocol+"://"+this.get("host")+":"+this.get("port");
 	this.boot.pathPrefix = this.get("path-prefix") || "";
-	// Complete initialisation of server extensions
-	this.invokeExtensionHook("server-completed-initialisation");
 }
 
-Server.prototype.invokeExtensionHook = function(hookName) {
-	$tw.utils.each(this.extensions,function(extension) {
-		extension.hook(hookName);
-	});
-};
-
 /*
 Send a response to the client. This method checks if the response must be sent
 or if the client alrady has the data cached. If that's the case only a 304
@@ -180,6 +162,12 @@ function sendResponse(request,response,statusCode,headers,data,encoding) {
 	response.end(data,encoding);
 }
 
+function redirect(request,response,statusCode,location) {
+	response.setHeader("Location",location);
+	response.statusCode = statusCode;
+	response.end()
+}
+
 /*
 Options include:
 cbPartStart(headers,name,filename) - invoked when a file starts being received
@@ -255,8 +243,8 @@ function streamMultipartData(request,options) {
 			} else {
 				const boundaryIndex = buffer.indexOf(boundaryBuffer);
 				if(boundaryIndex >= 0) {
-					// Return the part up to the boundary
+					// Return the part up to the boundary minus the terminating LF CR
-					options.cbPartChunk(Uint8Array.prototype.slice.call(buffer,0,boundaryIndex));
+					options.cbPartChunk(Uint8Array.prototype.slice.call(buffer,0,boundaryIndex - 2));
 					options.cbPartEnd();
 					processingPart = false;
 					buffer = Uint8Array.prototype.slice.call(buffer,boundaryIndex);
@@ -368,15 +356,10 @@ Server.prototype.requestHandler = function(request,response,options) {
 	state.queryParameters = querystring.parse(state.urlInfo.query);
 	state.pathPrefix = options.pathPrefix || this.get("path-prefix") || "";
 	state.sendResponse = sendResponse.bind(self,request,response);
+	state.redirect = redirect.bind(self,request,response);
 	state.streamMultipartData = streamMultipartData.bind(self,request);
 	// Get the principals authorized to access this resource
 	state.authorizationType = options.authorizationType || this.methodMappings[request.method] || "readers";
-	// Check for the CSRF header if this is a write
-	if(!this.csrfDisable && state.authorizationType === "writers" && request.headers["x-requested-with"] !== "TiddlyWiki") {
-		response.writeHead(403,"'X-Requested-With' header required to login to '" + this.servername + "'");
-		response.end();
-		return;
-	}
 	// Check whether anonymous access is granted
 	state.allowAnon = this.isAuthorized(state.authorizationType,null);
 	// Authenticate with the first active authenticator
@@ -406,6 +389,12 @@ Server.prototype.requestHandler = function(request,response,options) {
 		response.end();
 		return;
 	}
+	// If this is a write, check for the CSRF header unless globally disabled, or disabled for this route
+	if(!this.csrfDisable && !route.csrfDisable && state.authorizationType === "writers" && request.headers["x-requested-with"] !== "TiddlyWiki") {
+		response.writeHead(403,"'X-Requested-With' header required to login to '" + this.servername + "'");
+		response.end();
+		return;
+	}
 	// Receive the request body if necessary and hand off to the route handler
 	if(route.bodyFormat === "stream" || request.method === "GET" || request.method === "HEAD") {
 		// Let the route handle the request stream itself
@@ -466,10 +455,12 @@ Server.prototype.listen = function(port,host,prefix) {
 	}
 	// Create the server
 	var server = this.transport.createServer(this.listenOptions || {},function(request,response,options) {
-		var start = new Date().getTime()
+		if(self.get("debug-level") !== "none") {
+			var start = $tw.utils.timer();
 			response.on("finish",function() {
-			// console.log("Request",request.method,request.url,(new Date().getTime()) - start);
+				console.log("Response tim:",request.method,request.url,$tw.utils.timer() - start);
 			});	
+		}
 		self.requestHandler(request,response,options);
 	});
 	// Display the port number after we've started listening (the port number might have been specified as zero, in which case we will get an assigned port)

plugins/tiddlywiki/multiwikiserver/admin-ui/MultiWikiServer Administration.tid

@@ -38,19 +38,19 @@ title: MultiWikiServer Administration
 				<label class="mws-form-field-description">
 					Bag name
 				</label>
-				<$edit-text tiddler="$:/state/NewBagName" tag="input" placeholder="(bag name)" class="mws-form-field-input"/>
+				<$edit-text tiddler="$:/temp/NewBagName" tag="input" placeholder="(bag name)" class="mws-form-field-input"/>
 			</div>
 			<div class="mws-form-field">
 				<label class="mws-form-field-description">
 					Bag description
 				</label>
-				<$edit-text tiddler="$:/state/NewBagDescription" tag="input" placeholder="(description)" class="mws-form-field-input"/>
+				<$edit-text tiddler="$:/temp/NewBagDescription" tag="input" placeholder="(description)" class="mws-form-field-input"/>
 			</div>
 		</div>
 		<div class="mws-form-status">
-			<%if [[$:/state/NewBagError]get[text]else[]!match[]] %>
+			<%if [[$:/temp/NewBagError]get[text]else[]!match[]] %>
 				<div class="mws-form-error">
-					<$text text={{$:/state/NewBagError}}/>
+					<$text text={{$:/temp/NewBagError}}/>
 				</div>
 			<%endif%>
 		</div>
@@ -58,9 +58,9 @@ title: MultiWikiServer Administration
 			<$button class="mws-form-button">
 				<$transclude
 					$variable="createBag"
-					name={{$:/state/NewBagName}}
+					name={{$:/temp/NewBagName}}
-					description={{$:/state/NewBagDescription}}
+					description={{$:/temp/NewBagDescription}}
-					errorTiddler="$:/state/NewBagError"
+					errorTiddler="$:/temp/NewBagError"
 				/>
 				Create Bag
 			</$button>
@@ -103,25 +103,25 @@ title: MultiWikiServer Administration
 				<label class="mws-form-field-description">
 					Recipe name
 				</label>
-				<$edit-text tiddler="$:/state/NewRecipeName" tag="input" placeholder="(recipe name)" class="mws-form-field-input"/>
+				<$edit-text tiddler="$:/temp/NewRecipeName" tag="input" placeholder="(recipe name)" class="mws-form-field-input"/>
 			</div>
 			<div class="mws-form-field">
 				<label class="mws-form-field-description">
 					Bag names
 				</label>
-				<$edit-text tiddler="$:/state/NewRecipeBagNames" tag="input" placeholder="(space separated list of bags)"/>
+				<$edit-text tiddler="$:/temp/NewRecipeBagNames" tag="input" placeholder="(space separated list of bags)"/>
 			</div>
 			<div class="mws-form-field">
 				<label class="mws-form-field-description">
 					Recipe description
 				</label>
-				<$edit-text tiddler="$:/state/NewRecipeDescription" tag="input" placeholder="(description)" class="mws-form-field-input"/>
+				<$edit-text tiddler="$:/temp/NewRecipeDescription" tag="input" placeholder="(description)" class="mws-form-field-input"/>
 			</div>
 		</div>
 		<div class="mws-form-status">
-			<%if [[$:/state/NewRecipeError]get[text]else[]!match[]] %>
+			<%if [[$:/temp/NewRecipeError]get[text]else[]!match[]] %>
 				<div class="mws-form-error">
-					<$text text={{$:/state/NewRecipeError}}/>
+					<$text text={{$:/temp/NewRecipeError}}/>
 				</div>
 			<%endif%>
 		</div>
@@ -129,10 +129,10 @@ title: MultiWikiServer Administration
 			<$button class="mws-form-button">
 				<$transclude
 					$variable="createRecipe"
-					name={{$:/state/NewRecipeName}}
+					name={{$:/temp/NewRecipeName}}
-					bag_names={{$:/state/NewRecipeBagNames}}
+					bag_names={{$:/temp/NewRecipeBagNames}}
-					description={{$:/state/NewRecipeDescription}}
+					description={{$:/temp/NewRecipeDescription}}
-					errorTiddler="$:/state/NewRecipeError"
+					errorTiddler="$:/temp/NewRecipeError"
 				/>
 				Create Recipe
 			</$button>

plugins/tiddlywiki/multiwikiserver/config/MultiWikiServerAttachmentSizeLimit.tid

@@ -0,0 +1,2 @@
+title: $:/config/MultiWikiServer/AttachmentSizeLimit
+text: 204800

plugins/tiddlywiki/multiwikiserver/modules/attachment-store.js

@@ -0,0 +1,140 @@
+/*\
+title: $:/plugins/tiddlywiki/multiwikiserver/attachment-store.js
+type: application/javascript
+module-type: library
+
+Class to handle the attachments in the filing system
+
+The store folder looks like this:
+
+store/
+	inbox/ - files that are in the process of being uploaded via a multipart form upload
+		202402282125432742-0/
+			part000
+			part001
+			...
+		...
+	files/ - files that are the text content of large tiddlers
+		b7def178-79c4-4d88-b7a4-39763014a58b/
+			data.jpg - the extension is provided for convenience when directly inspecting the file system
+			meta.json - contains:
+				{
+					"filename": "data.jpg",
+					"type": "video/mp4",
+					"uploaded": "2024021821224823"
+				}
+	database.sql - The database file (managed by sql-tiddler-database.js)
+
+\*/
+
+(function() {
+
+/*
+Class to handle an attachment store. Options include:
+
+storePath - path to the store
+*/
+function AttachmentStore(options) {
+	options = options || {};
+	this.storePath = options.storePath;
+}
+
+/*
+Check if an attachment name is valid
+*/
+AttachmentStore.prototype.isValidAttachmentName = function(attachmentname) {
+    const re = new RegExp('^[a-f0-9]{64}$');
+	return re.test(attachmentname);
+};
+
+/*
+Saves an attachment to a file. Options include:
+
+text: text content (may be binary)
+type: MIME type of content
+reference: reference to use for debugging
+*/
+AttachmentStore.prototype.saveAttachment = function(options) {
+	const path = require("path"),
+		fs = require("fs");
+	// Compute the content hash for naming the attachment
+	const contentHash = $tw.sjcl.codec.hex.fromBits($tw.sjcl.hash.sha256.hash(options.text)).slice(0,64).toString();
+	// Choose the best file extension for the attachment given its type
+	const contentTypeInfo = $tw.config.contentTypeInfo[options.type] || $tw.config.contentTypeInfo["application/octet-stream"];
+	// Creat the attachment directory
+	const attachmentPath = path.resolve(this.storePath,"files",contentHash);
+	$tw.utils.createDirectory(attachmentPath);
+	// Save the data file
+	const dataFilename = "data" + contentTypeInfo.extension;
+	fs.writeFileSync(path.resolve(attachmentPath,dataFilename),options.text,contentTypeInfo.encoding);
+	// Save the meta.json file
+	fs.writeFileSync(path.resolve(attachmentPath,"meta.json"),JSON.stringify({
+		modified: $tw.utils.stringifyDate(new Date()),
+		contentHash: contentHash,
+		filename: dataFilename,
+		type: options.type
+	},null,4));
+	return contentHash;
+};
+
+/*
+Adopts an attachment file into the store
+*/
+AttachmentStore.prototype.adoptAttachment = function(incomingFilepath,type,hash) {
+	const path = require("path"),
+		fs = require("fs");
+	// Choose the best file extension for the attachment given its type
+	const contentTypeInfo = $tw.config.contentTypeInfo[type] || $tw.config.contentTypeInfo["application/octet-stream"];
+	// Creat the attachment directory
+	const attachmentPath = path.resolve(this.storePath,"files",hash);
+	$tw.utils.createDirectory(attachmentPath);
+	// Rename the data file
+	const dataFilename = "data" + contentTypeInfo.extension,
+		dataFilepath = path.resolve(attachmentPath,dataFilename);
+	fs.renameSync(incomingFilepath,dataFilepath);
+	// Save the meta.json file
+	fs.writeFileSync(path.resolve(attachmentPath,"meta.json"),JSON.stringify({
+		modified: $tw.utils.stringifyDate(new Date()),
+		contentHash: hash,
+		filename: dataFilename,
+		type: type
+	},null,4));
+	return hash;
+};
+
+/*
+Get an attachment ready to stream. Returns null if there is an error or:
+stream: filestream of file
+type: type of file
+*/
+AttachmentStore.prototype.getAttachmentStream = function(attachmentname) {
+	const path = require("path"),
+		fs = require("fs");
+	// Check the attachment name
+	if(this.isValidAttachmentName(attachmentname)) {
+		// Construct the path to the attachment directory
+		const attachmentPath = path.resolve(this.storePath,"files",attachmentname);
+		// Read the meta.json file
+		const metaJsonPath = path.resolve(attachmentPath,"meta.json");
+		if(fs.existsSync(metaJsonPath) && fs.statSync(metaJsonPath).isFile()) {
+			const meta = $tw.utils.parseJSONSafe(fs.readFileSync(metaJsonPath,"utf8"),function() {return null;});
+			if(meta) {
+				const dataFilepath = path.resolve(attachmentPath,meta.filename);
+				// Check if the data file exists
+				if(fs.existsSync(dataFilepath) && fs.statSync(dataFilepath).isFile()) {
+					// Stream the file
+					return {
+						stream: fs.createReadStream(dataFilepath),
+						type: meta.type
+					};
+				}
+			}
+		}
+	}
+	// An error occured
+	return null;
+};
+
+exports.AttachmentStore = AttachmentStore;
+
+})();

plugins/tiddlywiki/multiwikiserver/modules/init.js

@@ -21,14 +21,18 @@ exports.synchronous = true;
 exports.startup = function() {
 	var path = require("path");
 	// Create and initialise the tiddler store and upload manager
-	var SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore,
+	var AttachmentStore = require("$:/plugins/tiddlywiki/multiwikiserver/attachment-store.js").AttachmentStore,
+		attachmentStore = new AttachmentStore({
+			storePath: path.resolve($tw.boot.wikiPath,"store/")
+		}),
+		SqlTiddlerStore = require("$:/plugins/tiddlywiki/multiwikiserver/sql-tiddler-store.js").SqlTiddlerStore,
 		store = new SqlTiddlerStore({
 			databasePath: path.resolve($tw.boot.wikiPath,"store/database.sqlite"),
-			engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better") // better || wasm
+			engine: $tw.wiki.getTiddlerText("$:/config/MultiWikiServer/Engine","better"), // better || wasm
+			attachmentStore: attachmentStore
 		}),
 		MultipartFormManager = require("$:/plugins/tiddlywiki/multiwikiserver/multipart-form-manager.js").MultipartFormManager,
 		multipartFormManager = new MultipartFormManager({
-			inboxPath: path.resolve($tw.boot.wikiPath,"store/inbox"),
 			store: store
 		});
 	$tw.mws = {

plugins/tiddlywiki/multiwikiserver/modules/multipart-form-manager.js

@@ -15,13 +15,10 @@ be moved out of the store/inbox folder.
 /*
 Create an instance of the upload manager. Options include:
 
-inboxPath - path to the inbox folder
 store - sqlTiddlerStore to use for saving tiddlers
 */
 function MultipartFormManager(options) {
-	const path = require("path");
 	options = options || {};
-	this.inboxPath = options.inboxPath;
 	this.store = options.store;
 }
 

plugins/tiddlywiki/multiwikiserver/modules/route-delete-recipe-tiddler.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-delete-recipe-tiddler.js
 type: application/javascript
 module-type: route
 
-DELETE /wikis/:recipe_name/recipes/:bag_name/tiddler/:title
+DELETE /wiki/:recipe_name/recipes/:bag_name/tiddler/:title
 
 NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 

plugins/tiddlywiki/multiwikiserver/modules/route-get-bag-tiddler-blob.js

@@ -0,0 +1,38 @@
+/*\
+title: $:/plugins/tiddlywiki/multiwikiserver/route-get-bag-tiddler-blob.js
+type: application/javascript
+module-type: route
+
+GET /wiki/:bag_name/bags/:bag_name/tiddler/:title/blob
+
+\*/
+(function() {
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.method = "GET";
+
+exports.path = /^\/wiki\/([^\/]+)\/bags\/([^\/]+)\/tiddlers\/([^\/]+)\/blob$/;
+
+exports.handler = function(request,response,state) {
+	// Get the  parameters
+	const bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
+		bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]),
+		title = $tw.utils.decodeURIComponentSafe(state.params[2]);
+	if(bag_name === bag_name_2) {
+		const result = $tw.mws.store.getBagTiddlerStream(title,bag_name);
+		if(result) {
+			response.writeHead(200, "OK",{
+				"Content-Type":  result.type
+			});
+			result.stream.pipe(response);
+			return;
+		}
+	}
+	response.writeHead(404);
+	response.end();
+};
+
+}());

plugins/tiddlywiki/multiwikiserver/modules/route-get-bag-tiddler.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-bag-tiddler.js
 type: application/javascript
 module-type: route
 
-GET /wikis/:bag_name/bags/:bag_name/tiddler/:title
+GET /wiki/:bag_name/bags/:bag_name/tiddler/:title
 
 NOTE: Urls currently include the bag name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 
@@ -40,21 +40,22 @@ exports.handler = function(request,response,state) {
 				}
 			});
 			tiddlerFields.type = tiddlerFields.type || "text/vnd.tiddlywiki";
-			tiddlerFields = $tw.mws.store.processCanonicalUriTiddler(tiddlerFields,bag_name,null);
 			state.sendResponse(200,{"Content-Type": "application/json"},JSON.stringify(tiddlerFields),"utf8");
+			return;
 		} else {
 			// This is not a JSON API request, we should return the raw tiddler content
-			var type = result.tiddler.type || "text/plain";
+			const result = $tw.mws.store.getBagTiddlerStream(title,bag_name);
+			if(result) {
 				response.writeHead(200, "OK",{
-				"Content-Type":  type
+					"Content-Type":  result.type
 				});
-			response.write(result.tiddler.text || "",($tw.config.contentTypeInfo[type] ||{encoding: "utf8"}).encoding);
+				result.stream.pipe(response);
-			response.end();;
+				return;
+			}
+		}
 	}
-	} else {
 	response.writeHead(404);
 	response.end();
-	}
 };
 
 }());

plugins/tiddlywiki/multiwikiserver/modules/route-get-bag.js

@@ -3,7 +3,8 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-bag.js
 type: application/javascript
 module-type: route
 
-GET /wikis/:bag_name/bags/:bag_name
+GET /wiki/:bag_name/bags/:bag_name/
+GET /wiki/:bag_name/bags/:bag_name
 
 NOTE: Urls currently include the bag name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 
@@ -16,9 +17,14 @@ NOTE: Urls currently include the bag name twice. This is temporary to minimise t
 
 exports.method = "GET";
 
-exports.path = /^\/wiki\/([^\/]+)\/bags\/(.+)$/;
+exports.path = /^\/wiki\/([^\/]+)\/bags\/([^\/]+)(\/?)$/;
 
 exports.handler = function(request,response,state) {
+	// Redirect if there is no trailing slash. We do this so that the relative URL specified in the upload form works correctly
+	if(state.params[2] !== "/") {
+		state.redirect(301,state.urlInfo.path + "/");
+		return;
+	}
 	// Get the  parameters
 	var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
 		bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]),
@@ -40,7 +46,7 @@ exports.handler = function(request,response,state) {
 				<body>
 			`);
 			// Render the html
-			var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/get-bags",{
+			var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/get-bag",{
 				variables: {
 					"bag-name": bag_name,
 					"bag-titles": JSON.stringify(titles)
@@ -51,7 +57,7 @@ exports.handler = function(request,response,state) {
 				</body>
 				</html>
 			`);
-			response.end();;
+			response.end();
 		}
 	} else {
 		response.writeHead(404);

plugins/tiddlywiki/multiwikiserver/modules/route-get-recipe-tiddler.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-recipe-tiddler.js
 type: application/javascript
 module-type: route
 
-GET /wikis/:recipe_name/recipes/:recipe_name/tiddler/:title
+GET /wiki/:recipe_name/recipes/:recipe_name/tiddler/:title
 
 NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 
@@ -40,7 +40,6 @@ exports.handler = function(request,response,state) {
 				}
 			});
 			tiddlerFields.type = tiddlerFields.type || "text/vnd.tiddlywiki";
-			tiddlerFields = $tw.mws.store.processCanonicalUriTiddler(tiddlerFields,null,recipe_name);
 			state.sendResponse(200,{"Content-Type": "application/json"},JSON.stringify(tiddlerFields),"utf8");
 		} else {
 			// This is not a JSON API request, we should return the raw tiddler content

plugins/tiddlywiki/multiwikiserver/modules/route-get-recipe-tiddlers-json.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-recipe-tiddlers-json.js
 type: application/javascript
 module-type: route
 
-PUT /wikis/:recipe_name/recipes/:recipe_name/tiddlers.json?filter=:filter
+PUT /wiki/:recipe_name/recipes/:recipe_name/tiddlers.json?filter=:filter
 
 NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 

plugins/tiddlywiki/multiwikiserver/modules/route-get-recipe.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-recipe.js
 type: application/javascript
 module-type: route
 
-GET /wikis/:recipe_name
+GET /wiki/:recipe_name
 
 \*/
 (function() {
@@ -18,15 +18,15 @@ exports.path = /^\/wiki\/([^\/]+)$/;
 
 exports.handler = function(request,response,state) {
 	// Get the recipe name from the parameters
-	var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]);
+	var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
+		recipeTiddlers = recipe_name && $tw.mws.store.getRecipeTiddlers(recipe_name);
 	// Check request is valid
-	if(recipe_name) {
+	if(recipe_name && recipeTiddlers) {
 		// Start the response
 		response.writeHead(200, "OK",{
 			"Content-Type": "text/html"
 		});
 		// Get the tiddlers in the recipe
-		var recipeTiddlers = $tw.mws.store.getRecipeTiddlers(recipe_name);
 		// Render the template
 		var template = $tw.mws.store.adminWiki.renderTiddler("text/plain","$:/core/templates/tiddlywiki5.html",{
 			variables: {
@@ -53,7 +53,6 @@ exports.handler = function(request,response,state) {
 			var result = $tw.mws.store.getRecipeTiddler(recipeTiddlerInfo.title,recipe_name);
 			if(result) {
 				var tiddlerFields = result.tiddler;
-				tiddlerFields = $tw.mws.store.processCanonicalUriTiddler(tiddlerFields,null,recipe_name);
 				response.write(JSON.stringify(tiddlerFields).replace(/</g,"\\u003c"));
 				response.write(",\n")
 			}

plugins/tiddlywiki/multiwikiserver/modules/route-get-status.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-get-status.js
 type: application/javascript
 module-type: route
 
-GET /wikis/:recipe_name/status
+GET /wiki/:recipe_name/status
 
 \*/
 (function() {

plugins/tiddlywiki/multiwikiserver/modules/route-post-bag-tiddlers.js

@@ -0,0 +1,137 @@
+/*\
+title: $:/plugins/tiddlywiki/multiwikiserver/route-post-bag-tiddlers.js
+type: application/javascript
+module-type: route
+
+POST /wiki/:bag_name/bags/:bag_name/tiddlers/
+POST /wiki/:bag_name/bags/:bag_name/tiddlers
+
+NOTE: Urls currently include the bag name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
+
+\*/
+(function() {
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+exports.method = "POST";
+
+exports.path = /^\/wiki\/([^\/]+)\/bags\/([^\/]+)\/tiddlers\/$/;
+
+exports.bodyFormat = "stream";
+
+exports.csrfDisable = true;
+
+exports.handler = function(request,response,state) {
+	const path = require("path"),
+		fs = require("fs");
+	// Get the  parameters
+	var bag_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
+		bag_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]);
+console.log(`Got ${bag_name} and ${bag_name_2}`)
+	// Require the recipe names to match
+	if(bag_name !== bag_name_2) {
+		return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: bag names do not match");
+	}
+	// Process the incoming data
+	const inboxName = $tw.utils.stringifyDate(new Date());
+	const inboxPath = path.resolve($tw.mws.store.attachmentStore.storePath,"inbox",inboxName);
+	$tw.utils.createDirectory(inboxPath);
+	let fileStream = null; // Current file being written
+	let hash = null; // Accumulating hash of current part
+	let length = 0; // Accumulating length of current part
+	const parts = [];
+	state.streamMultipartData({
+		cbPartStart: function(headers,name,filename) {
+			console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
+			const part = {
+				name: name,
+				filename: filename,
+				headers: headers
+			};
+			if(filename) {
+				const inboxFilename = (parts.length).toString();
+				part.inboxFilename = path.resolve(inboxPath,inboxFilename);
+				fileStream = fs.createWriteStream(part.inboxFilename);
+			} else {
+				part.value = "";
+			}
+			hash = new $tw.sjcl.hash.sha256();
+			length = 0;
+			parts.push(part)
+		},
+		cbPartChunk: function(chunk) {
+			if(fileStream) {
+				fileStream.write(chunk);
+			} else {
+				parts[parts.length - 1].value += chunk;
+			}
+			length = length + chunk.length;
+			hash.update(chunk);
+			console.log(`Got a chunk of length ${chunk.length}, length is now ${length}`);
+		},
+		cbPartEnd: function() {
+			if(fileStream) {
+				fileStream.end();
+			}
+			fileStream = null;
+			parts[parts.length - 1].hash = $tw.sjcl.codec.hex.fromBits(hash.finalize()).slice(0,64).toString();
+			hash = null;
+		},
+		cbFinished: function(err) {
+			if(err) {
+				state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
+			} else {
+				console.log(`Multipart form data processed as ${JSON.stringify(parts,null,4)}`);
+				const partFile = parts.find(part => part.name === "file-to-upload" && !!part.filename);
+				if(!partFile) {
+					return state.sendResponse(400, {"Content-Type": "text/plain"},"Missing file to upload");
+				}
+				const type = partFile.headers["content-type"];
+				const tiddlerFields = {
+					title: partFile.filename,
+					type: type
+				};
+				for(const part of parts) {
+					const tiddlerFieldPrefix = "tiddler-field-";
+					if(part.name.startsWith(tiddlerFieldPrefix)) {
+						tiddlerFields[part.name.slice(tiddlerFieldPrefix.length)] = part.value.trim();
+					}
+				}
+				console.log(`Creating tiddler with ${JSON.stringify(tiddlerFields)} and ${partFile.filename}`)
+				$tw.mws.store.saveBagTiddlerWithAttachment(tiddlerFields,bag_name,{
+					filepath: partFile.inboxFilename,
+					type: type,
+					hash: partFile.hash
+				});
+				$tw.utils.deleteDirectory(inboxPath);
+				response.writeHead(200, "OK",{
+					"Content-Type":  "text/html"
+				});
+				response.write(`
+					<!doctype html>
+					<head>
+						<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+					</head>
+					<body>
+				`);
+				// Render the html
+				var html = $tw.mws.store.adminWiki.renderTiddler("text/html","$:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers",{
+					variables: {
+						"bag-name": bag_name,
+						"imported-titles": JSON.stringify([tiddlerFields.title])
+					}
+				});
+				response.write(html);
+				response.write(`
+					</body>
+					</html>
+				`);
+				response.end();
+			}
+		}
+	});
+};
+
+}());

plugins/tiddlywiki/multiwikiserver/modules/route-post-recipe-tiddlers.js

@@ -1,71 +0,0 @@
-/*\
-title: $:/plugins/tiddlywiki/multiwikiserver/route-post-recipe-tiddlers.js
-type: application/javascript
-module-type: route
-
-POST /wikis/:recipe_name/recipes/:recipe_name/tiddlers
-
-NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
-
-\*/
-(function() {
-
-/*jslint node: true, browser: true */
-/*global $tw: false */
-"use strict";
-
-exports.method = "POST";
-
-exports.path = /^\/wiki\/([^\/]+)\/recipes\/([^\/]+)\/tiddlers$/;
-
-exports.bodyFormat = "stream";
-
-exports.handler = function(request,response,state) {
-	const fs = require("fs");
-	// Get the  parameters
-	var recipe_name = $tw.utils.decodeURIComponentSafe(state.params[0]),
-		recipe_name_2 = $tw.utils.decodeURIComponentSafe(state.params[1]);
-console.log(`Got to here ${recipe_name} and ${recipe_name_2}`)
-	// Require the recipe names to match
-	if(recipe_name !== recipe_name_2) {
-		return state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: recipe names do not match");
-	}
-	// Process the incoming data
-	let fileStream = null;
-	let fieldValue = "";
-	state.streamMultipartData({
-		cbPartStart: function(headers,name,filename) {
-			console.log(`Received file ${name} and ${filename} with ${JSON.stringify(headers)}`)
-			if(filename) {
-				fileStream = fs.createWriteStream(filename);
-			} else {
-				fieldValue = "";
-			}
-		},
-		cbPartChunk: function(chunk) {
-			if(fileStream) {
-				fileStream.write(chunk);
-			} else {
-				fieldValue = fieldValue + chunk;
-			}
-		},
-		cbPartEnd: function() {
-			if(fileStream) {
-				fileStream.end();
-				fileStream = null;
-			} else {
-				console.log("Data was " + fieldValue);
-				fieldValue = "";
-			}
-		},
-		cbFinished: function(err) {
-			if(err) {
-				state.sendResponse(400,{"Content-Type": "text/plain"},"Bad Request: " + err);
-			} else {
-				state.sendResponse(200, {"Content-Type": "text/plain"},"Multipart data processed");
-			}
-		}
-	});
-};
-
-}());

plugins/tiddlywiki/multiwikiserver/modules/route-put-bag.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-put-bag.js
 type: application/javascript
 module-type: route
 
-PUT /wikis/:bag_name/bags/:bag_name
+PUT /wiki/:bag_name/bags/:bag_name
 
 NOTE: Urls currently include the bag name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 

plugins/tiddlywiki/multiwikiserver/modules/route-put-recipe-tiddler.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-put-recipe-tiddler.js
 type: application/javascript
 module-type: route
 
-PUT /wikis/:recipe_name/recipes/:recipe_name/tiddlers/:title
+PUT /wiki/:recipe_name/recipes/:recipe_name/tiddlers/:title
 
 NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 

plugins/tiddlywiki/multiwikiserver/modules/route-put-recipe.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/multiwikiserver/route-put-recipe.js
 type: application/javascript
 module-type: route
 
-PUT /wikis/:recipe_name/recipes/:recipe_name
+PUT /wiki/:recipe_name/recipes/:recipe_name
 
 NOTE: Urls currently include the recipe name twice. This is temporary to minimise the changes to the TiddlyWeb plugin
 

plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-database.js

@@ -133,6 +133,7 @@ SqlTiddlerDatabase.prototype.createTables = function() {
 			tiddler_id INTEGER PRIMARY KEY AUTOINCREMENT,
 			bag_id INTEGER NOT NULL,
 			title TEXT NOT NULL,
+			attachment_blob TEXT, -- null or the name of an attachment blob
 			FOREIGN KEY (bag_id) REFERENCES bags(bag_id) ON UPDATE CASCADE ON DELETE CASCADE,
 			UNIQUE (bag_id, title)
 		)
@@ -253,16 +254,19 @@ SqlTiddlerDatabase.prototype.createRecipe = function(recipename,bagnames,descrip
 /*
 Returns {tiddler_id:}
 */
-SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
+SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname,attachment_blob) {
+	attachment_blob = attachment_blob || null;
 	// Update the tiddlers table
 	var info = this.runStatement(`
-		INSERT OR REPLACE INTO tiddlers (bag_id, title)
+		INSERT OR REPLACE INTO tiddlers (bag_id, title, attachment_blob)
 		VALUES (
 			(SELECT bag_id FROM bags WHERE bag_name = $bag_name),
-			$title
+			$title,
+			$attachment_blob
 		)
 	`,{
 		$title: tiddlerFields.title,
+		$attachment_blob: attachment_blob,
 		$bag_name: bagname
 	});
 	// Update the fields table
@@ -295,7 +299,7 @@ SqlTiddlerDatabase.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
 /*
 Returns {tiddler_id:,bag_name:} or null if the recipe is empty
 */
-SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipename) {
+SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipename,attachment_blob) {
 	// Find the topmost bag in the recipe
 	var row = this.runStatementGet(`
 		SELECT b.bag_name
@@ -319,7 +323,7 @@ SqlTiddlerDatabase.prototype.saveRecipeTiddler = function(tiddlerFields,recipena
 		return null;
 	}
 	// Save the tiddler to the topmost bag
-	var info = this.saveBagTiddler(tiddlerFields,row.bag_name);
+	var info = this.saveBagTiddler(tiddlerFields,row.bag_name,attachment_blob);
 	return {
 		tiddler_id: info.tiddler_id,
 		bag_name: row.bag_name
@@ -354,27 +358,34 @@ SqlTiddlerDatabase.prototype.deleteTiddler = function(title,bagname) {
 };
 
 /*
-returns {tiddler_id:,tiddler:}
+returns {tiddler_id:,tiddler:,attachment_blob:}
 */
 SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) {
-	const rows = this.runStatementGetAll(`
+	const rowTiddler = this.runStatementGet(`
-		SELECT field_name, field_value, tiddler_id
+		SELECT t.tiddler_id, t.attachment_blob
-		FROM fields
-		WHERE tiddler_id = (
-			SELECT t.tiddler_id
 		FROM bags AS b
 		INNER JOIN tiddlers AS t ON b.bag_id = t.bag_id
 		WHERE t.title = $title AND b.bag_name = $bag_name
-		)
 	`,{
 		$title: title,
 		$bag_name: bagname
 	});
+	if(!rowTiddler) {
+		return null;
+	}
+	const rows = this.runStatementGetAll(`
+		SELECT field_name, field_value, tiddler_id
+		FROM fields
+		WHERE tiddler_id = $tiddler_id
+	`,{
+		$tiddler_id: rowTiddler.tiddler_id
+	});
 	if(rows.length === 0) {
 		return null;
 	} else {
 		return {
 			tiddler_id: rows[0].tiddler_id,
+			attachment_blob: rowTiddler.attachment_blob,
 			tiddler: rows.reduce((accumulator,value) => {
 					accumulator[value["field_name"]] = value.field_value;
 					return accumulator;
@@ -384,11 +395,11 @@ SqlTiddlerDatabase.prototype.getBagTiddler = function(title,bagname) {
 };
 
 /*
-Returns {bag_name:, tiddler: {fields}, tiddler_id:}
+Returns {bag_name:, tiddler: {fields}, tiddler_id:, attachment_blob:}
 */
 SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
 	const rowTiddlerId = this.runStatementGet(`	
-		SELECT t.tiddler_id, b.bag_name
+		SELECT t.tiddler_id, t.attachment_blob, b.bag_name
 		FROM bags AS b
 		INNER JOIN recipe_bags AS rb ON b.bag_id = rb.bag_id
 		INNER JOIN recipes AS r ON rb.recipe_id = r.recipe_id
@@ -415,6 +426,7 @@ SqlTiddlerDatabase.prototype.getRecipeTiddler = function(title,recipename) {
 	return {
 		bag_name: rowTiddlerId.bag_name,
 		tiddler_id: rowTiddlerId.tiddler_id,
+		attachment_blob: rowTiddlerId.attachment_blob,
 		tiddler: rows.reduce((accumulator,value) => {
 				accumulator[value["field_name"]] = value.field_value;
 				return accumulator;
@@ -442,9 +454,17 @@ SqlTiddlerDatabase.prototype.getBagTiddlers = function(bagname) {
 };
 
 /*
-Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns an empty array for recipes that do not exist
+Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist
 */
 SqlTiddlerDatabase.prototype.getRecipeTiddlers = function(recipename) {
+	const rowsCheckRecipe = this.runStatementGetAll(`
+		SELECT * FROM recipes WHERE recipes.recipe_name = $recipe_name
+	`,{
+		$recipe_name: recipename
+	});
+	if(rowsCheckRecipe.length === 0) {
+		return null;
+	}
 	const rows = this.runStatementGetAll(`
 		SELECT title, bag_name
 		FROM (

plugins/tiddlywiki/multiwikiserver/modules/sql-tiddler-store.js

@@ -9,7 +9,7 @@ This class is largely a wrapper for the sql-tiddler-database.js class, adding th
 
 * Validating requests (eg bag and recipe name constraints)
 * Synchronising bag and recipe names to the admin wiki
-* Handling _canonical_uri tiddlers
+* Handling large tiddlers as attachments
 
 \*/
 
@@ -20,10 +20,12 @@ Create a tiddler store. Options include:
 
 databasePath - path to the database file (can be ":memory:" to get a temporary database)
 adminWiki - reference to $tw.Wiki object into which entity state tiddlers should be saved
+attachmentStore - reference to associated attachment store
 engine - wasm | better
 */
 function SqlTiddlerStore(options) {
 	options = options || {};
+	this.attachmentStore = options.attachmentStore;
 	this.adminWiki = options.adminWiki || $tw.wiki;
 	this.entityStateTiddlerPrefix = "$:/state/MultiWikiServer/";
 	// Create the database
@@ -114,31 +116,43 @@ Given tiddler fields, tiddler_id and a bagname, return the tiddler fields after
 - Apply the tiddler_id as the revision field
 - Apply the bag_name as the bag field
 */
-SqlTiddlerStore.prototype.processOutgoingTiddler = function(tiddlerFields,tiddler_id,bag_name) {
+SqlTiddlerStore.prototype.processOutgoingTiddler = function(tiddlerFields,tiddler_id,bag_name,attachment_blob) {
-	return Object.assign({},tiddlerFields,{
+	const fields = Object.assign({},tiddlerFields,{
 		revision: "" + tiddler_id,
 		bag: bag_name
 	});
+	if(attachment_blob !== null) {
+		delete fields.text;
+		fields._canonical_uri = `/wiki/${encodeURIComponent(bag_name)}/bags/${encodeURIComponent(bag_name)}/tiddlers/${encodeURIComponent(tiddlerFields.title)}/blob`;
+	}
+	return fields;
 };
 
 /*
-Given tiddler fields and a bagname or a recipename, if the text field is over a threshold, modify
-the tiddler to use _canonical_uri, otherwise return the tiddler unmodified
 */
-SqlTiddlerStore.prototype.processCanonicalUriTiddler = function(tiddlerFields,bag_name,recipe_name) {
+SqlTiddlerStore.prototype.processIncomingTiddler = function(tiddlerFields) {
-	if((tiddlerFields.text || "").length > 10 * 1024 * 1024) {
+	let attachmentSizeLimit = $tw.utils.parseNumber(this.adminWiki.getTiddlerText("$:/config/MultiWikiServer/AttachmentSizeLimit"));
-		return Object.assign({},tiddlerFields,{
+	if(attachmentSizeLimit < 100 * 1024) {
-			text: undefined,
+		attachmentSizeLimit = 100 * 1024;
-			_canonical_uri: recipe_name
+	}
-				? `/wiki/${recipe_name}/recipes/${recipe_name}/tiddlers/${tiddlerFields.title}`
+	if(tiddlerFields.text && tiddlerFields.text.length > attachmentSizeLimit) {
-				: `/wiki/${bag_name}/bags/${bag_name}/tiddlers/${tiddlerFields.title}`
+		const attachment_blob = this.attachmentStore.saveAttachment({
+			text: tiddlerFields.text,
+			type: tiddlerFields.type,
+			reference: tiddlerFields.title
 		});
+		return {
+			tiddlerFields: Object.assign({},tiddlerFields,{text: undefined}),
+			attachment_blob: attachment_blob
+		};
 	} else {
-		return tiddlerFields;
+		return {
+			tiddlerFields: tiddlerFields,
+			attachment_blob: null
+		};
 	}
 };
 
-
 SqlTiddlerStore.prototype.saveTiddlersFromPath = function(tiddler_files_path,bag_name) {
 	var self = this;
 	this.sqlTiddlerDatabase.transaction(function() {
@@ -150,7 +164,7 @@ SqlTiddlerStore.prototype.saveTiddlersFromPath = function(tiddler_files_path,bag
 		// Save the tiddlers
 		for(const tiddlersFromFile of tiddlersFromPath) {
 			for(const tiddler of tiddlersFromFile.tiddlers) {
-				self.saveBagTiddler(tiddler,bag_name);
+				self.saveBagTiddler(tiddler,bag_name,null);
 			}
 		}
 	});
@@ -220,15 +234,37 @@ SqlTiddlerStore.prototype.createRecipe = function(recipename,bagnames,descriptio
 /*
 Returns {tiddler_id:}
 */
-SqlTiddlerStore.prototype.saveBagTiddler = function(tiddlerFields,bagname) {
+SqlTiddlerStore.prototype.saveBagTiddler = function(incomingTiddlerFields,bagname) {
-	return this.sqlTiddlerDatabase.saveBagTiddler(tiddlerFields,bagname);
+	const {tiddlerFields, attachment_blob} = this.processIncomingTiddler(incomingTiddlerFields);
+	return this.sqlTiddlerDatabase.saveBagTiddler(tiddlerFields,bagname,attachment_blob);
+};
+
+/*
+Create a tiddler in a bag adopting the specified file as the attachment. The attachment file must be on the same disk as the attachment store
+Options include:
+
+filepath - filepath to the attachment file
+hash - string hash of the attachment file
+type - content type of file as uploaded
+
+Returns {tiddler_id:}
+*/
+SqlTiddlerStore.prototype.saveBagTiddlerWithAttachment = function(incomingTiddlerFields,bagname,options) {
+	console.log(`saveBagTiddlerWithAttachment ${JSON.stringify(incomingTiddlerFields)}, ${bagname}, ${JSON.stringify(options)}`);
+	const attachment_blob = this.attachmentStore.adoptAttachment(options.filepath,options.type,options.hash);
+	if(attachment_blob) {
+		return this.sqlTiddlerDatabase.saveBagTiddler(incomingTiddlerFields,bagname,attachment_blob);
+	} else {
+		return null;
+	}
 };
 
 /*
 Returns {tiddler_id:,bag_name:}
 */
-SqlTiddlerStore.prototype.saveRecipeTiddler = function(tiddlerFields,recipename) {
+SqlTiddlerStore.prototype.saveRecipeTiddler = function(incomingTiddlerFields,recipename) {
-	return this.sqlTiddlerDatabase.saveRecipeTiddler(tiddlerFields,recipename);
+	const {tiddlerFields, attachment_blob} = this.processIncomingTiddler(incomingTiddlerFields);
+	return this.sqlTiddlerDatabase.saveRecipeTiddler(tiddlerFields,recipename,attachment_blob);
 };
 
 SqlTiddlerStore.prototype.deleteTiddler = function(title,bagname) {
@@ -245,13 +281,43 @@ SqlTiddlerStore.prototype.getBagTiddler = function(title,bagname) {
 			{},
 			tiddlerInfo,
 			{
-				tiddler: this.processOutgoingTiddler(tiddlerInfo.tiddler,tiddlerInfo.tiddler_id,bagname)
+				tiddler: this.processOutgoingTiddler(tiddlerInfo.tiddler,tiddlerInfo.tiddler_id,bagname,tiddlerInfo.attachment_blob)
 			});	
 	} else {
 		return null;
 	}
 };
 
+/*
+Get an attachment ready to stream. Returns null if there is an error or:
+stream: stream of file
+type: type of file
+*/
+SqlTiddlerStore.prototype.getBagTiddlerStream = function(title,bagname) {
+	const tiddlerInfo = this.sqlTiddlerDatabase.getBagTiddler(title,bagname);
+	if(tiddlerInfo) {
+		if(tiddlerInfo.attachment_blob) {
+			return this.attachmentStore.getAttachmentStream(tiddlerInfo.attachment_blob);
+		} else {
+			const { Readable } = require('stream');
+			const stream = new Readable();
+			stream._read = function() {
+				// Push data
+				const type = tiddlerInfo.tiddler.type || "text/plain";
+				stream.push(tiddlerInfo.tiddler.text || "",($tw.config.contentTypeInfo[type] ||{encoding: "utf8"}).encoding);
+				// Push null to indicate the end of the stream
+				stream.push(null);
+			};
+			return {
+				stream: stream,
+				type: tiddlerInfo.tiddler.type || "text/plain"
+			}
+		}
+	} else {
+		return null;
+	}
+};
+
 /*
 Returns {bag_name:, tiddler: {fields}, tiddler_id:}
 */
@@ -262,7 +328,7 @@ SqlTiddlerStore.prototype.getRecipeTiddler = function(title,recipename) {
 			{},
 			tiddlerInfo,
 			{
-				tiddler: this.processOutgoingTiddler(tiddlerInfo.tiddler,tiddlerInfo.tiddler_id,tiddlerInfo.bag_name)
+				tiddler: this.processOutgoingTiddler(tiddlerInfo.tiddler,tiddlerInfo.tiddler_id,tiddlerInfo.bag_name,tiddlerInfo.attachment_blob)
 			});
 	} else {
 		return null;
@@ -277,7 +343,7 @@ SqlTiddlerStore.prototype.getBagTiddlers = function(bagname) {
 };
 
 /*
-Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns an empty array for recipes that do not exist
+Get the titles of the tiddlers in a recipe as {title:,bag_name:}. Returns null for recipes that do not exist
 */
 SqlTiddlerStore.prototype.getRecipeTiddlers = function(recipename) {
 	return this.sqlTiddlerDatabase.getRecipeTiddlers(recipename);

plugins/tiddlywiki/multiwikiserver/templates/get-bag.tid

@@ -0,0 +1,51 @@
+title: $:/plugins/tiddlywiki/multiwikiserver/templates/get-bag
+
+! <$image
+	source=`/wiki/${ [<bag-name>encodeuricomponent[]] }$/bags/${ [<bag-name>encodeuricomponent[]] }$/tiddlers/%24%3A%2Ffavicon.ico`
+	class="mws-favicon-small"
+	width="32px"
+>
+	<$image
+		source="$:/plugins/multiwikiserver/images/missing-favicon.png"
+		class="mws-favicon-small"
+		width="32px"
+	/>
+</$image> Bag <$text text={{{ [<bag-name>]}}}/>
+
+<form
+	method="post"
+	action="tiddlers/"
+	enctype="multipart/form-data"
+>
+	<div>
+		<label>
+			File to upload:
+		</label>
+		<input type="file" name="file-to-upload" accept="*/*" />
+	</div>
+	<div>
+		<label>
+			Tiddler title:
+		</label>
+		<input type="text" name="tiddler-field-title" />
+	</div>
+	<div>
+		<label>
+			Tiddler tags:
+		</label>
+		<input type="text" name="tiddler-field-tags" />
+	</div>
+	<div>
+		<input type="submit" value="Upload"/>
+	</div>
+</form>
+
+<ul>
+<$list filter="[<bag-titles>jsonget[]sort[]]">
+<li>
+<a href=`/wiki/${ [<bag-name>encodeuricomponent[]] }$/bags/${ [<bag-name>encodeuricomponent[]] }$/tiddlers/${ [<currentTiddler>encodeuricomponent[]] }$` rel="noopener noreferrer" target="_blank">
+<$text text=<<currentTiddler>>/>
+</a>
+</li>
+</$list>
+</ul>

plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers.tid

@@ -1,4 +1,4 @@
-title: $:/plugins/tiddlywiki/multiwikiserver/templates/get-bags
+title: $:/plugins/tiddlywiki/multiwikiserver/templates/post-bag-tiddlers
 
 ! <$image
 	source=`/wiki/${ [<bag-name>encodeuricomponent[]] }$/bags/${ [<bag-name>encodeuricomponent[]] }$/tiddlers/%24%3A%2Ffavicon.ico`
@@ -12,8 +12,16 @@ title: $:/plugins/tiddlywiki/multiwikiserver/templates/get-bags
 	/>
 </$image> Bag <$text text={{{ [<bag-name>]}}}/>
 
+<p>
+Go back to <a href="..">Bag <$text text={{{ [<bag-name>]}}}/></a>
+</p>
+
+<p>
+The following tiddlers were successfully imported:
+</p>
+
 <ul>
-<$list filter="[<bag-titles>jsonget[]sort[]]">
+<$list filter="[<imported-titles>jsonget[]sort[]]">
 <li>
 <a href=`/wiki/${ [<bag-name>encodeuricomponent[]] }$/bags/${ [<bag-name>encodeuricomponent[]] }$/tiddlers/${ [<currentTiddler>encodeuricomponent[]] }$` rel="noopener noreferrer" target="_blank">
 <$text text=<<currentTiddler>>/>