Update HTML deserializer to cope with TiddlyWiki classic and TW5 files

core/modules/commands/load.js

@@ -35,8 +35,7 @@ Command.prototype.execute = function() {
 			self.callback(err);
 		} else {
 			var fields = {title: self.params[0]},
-				extname = path.extname(self.params[0]),
-				type = extname === ".html" ? "application/vnd.tiddlywiki2" : extname;
+				type = path.extname(self.params[0]);
 			var tiddlers = self.commander.wiki.deserializeTiddlers(type,data,fields);
 			if(!tiddlers) {
 				self.callback("No tiddlers found in file \"" + self.params[0] + "\"");

core/modules/deserializers.js

@@ -13,22 +13,29 @@ Functions to deserialise tiddlers from a block of text
 "use strict";
 
 /*
-Utility function to parse an old-style tiddler DIV. It looks like this:
+Utility function to parse an old-style tiddler DIV in a *.tid file. It looks like this:
 
 <div title="Title" creator="JoeBloggs" modifier="JoeBloggs" created="201102111106" modified="201102111310" tags="myTag [[my long tag]]">
 <pre>The text of the tiddler (without the expected HTML encoding).
 </pre>
 </div>
 
-Note that the field attributes are HTML encoded, but that the body of the <PRE> tag is not.
+Note that the field attributes are HTML encoded, but that the body of the <PRE> tag is not encoded.
+
+When these tiddler DIVs are encountered within a TiddlyWiki HTML file then the body is encoded in the usual way.
 */
-var parseTiddlerDiv = function(text,fields) {
+var parseTiddlerDiv = function(text /* [,fields] */) {
+	// Slot together the default results
 	var result = {};
-	if(fields) {
+	if(arguments.length > 1) {
+		for(var f=1; f<arguments.length; f++) {
+			var fields = arguments[f];
 			for(var t in fields) {
 				result[t] = fields[t];		
 			}
 		}
+	}
+	// Parse the DIV body
 	var divRegExp = /^\s*<div\s+([^>]*)>((?:.|\n)*)<\/div>\s*$/gi,
 		subDivRegExp = /^\s*<pre>((?:.|\n)*)<\/pre>\s*$/gi,
 		attrRegExp = /\s*([^=\s]+)\s*=\s*"([^"]*)"/gi,
@@ -49,8 +56,10 @@ var parseTiddlerDiv = function(text,fields) {
 				result[name] = value;
 			}
 		} while(attrMatch);
-	}
 		return result;
+	} else {
+		return undefined;
+	}
 };
 
 exports["application/x-tiddler-html-div"] = function(text,fields) {
@@ -75,34 +84,48 @@ exports["application/json"] = function(text,fields) {
 	return result;
 };
 
-exports["application/vnd.tiddlywiki2"] = function(text,fields) {
-	var locateStoreArea = function(tiddlywikidoc) {
-			var startSaveArea = '<div id="' + 'storeArea">',
-				startSaveAreaRegExp = /<div id=["']?storeArea['"]?>/gi,
-				endSaveArea = '</d' + 'iv>',
-				endSaveAreaCaps = '</D' + 'IV>',
-				posOpeningDiv = tiddlywikidoc.search(startSaveAreaRegExp),
-				limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-STOREAREA--"+">");
-			if(limitClosingDiv == -1) {
-				limitClosingDiv = tiddlywikidoc.indexOf("<"+"!--POST-BODY-START--"+">");
+/*
+Parse an HTML file into tiddlers. There are three possibilities:
+# A TiddlyWiki classic HTML file containing `application/vnd.tiddlywiki2` tiddlers
+# A TiddlyWiki5 HTML file containing `application/vnd.tiddlywiki` tiddlers
+# An ordinary HTML file
+*/
+exports["text/html"] = function(text,fields) {
+	// Check if we've got a store area
+	var storeAreaMarkerRegExp = /<div id=["']?storeArea['"]?( style=["']?display:none;["']?)?>/gi,
+		match = storeAreaMarkerRegExp.exec(text);
+	if(match) {
+		// If so, it's either a classic TiddlyWiki file or a TW5 file
+		return deserializeTiddlyWikiFile(text,storeAreaMarkerRegExp.lastIndex,!!match[1],fields);
+	} else {
+		// It's not a TiddlyWiki so we'll return the entire HTML file as a tiddler
+		return deserializeHtmlFile(text,fields);
 	}
-			var start = limitClosingDiv == -1 ? tiddlywikidoc.length : limitClosingDiv,
-				posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveArea,start);
-			if(posClosingDiv == -1) {
-				posClosingDiv = tiddlywikidoc.lastIndexOf(endSaveAreaCaps,start);
-			}
-			return (posOpeningDiv != -1 && posClosingDiv != -1) ? [posOpeningDiv + startSaveArea.length,posClosingDiv] : null;
-		},
-		results = [],
-		storeAreaPos = locateStoreArea(text);
-	if(storeAreaPos) {
-		var endOfDivRegExp = /(<\/div>\s*)/gi,
-			startPos = storeAreaPos[0];
+};
+
+function deserializeHtmlFile(text,fields) {
+	var result = {};
+	$tw.utils.each(fields,function(value,name) {
+		result[name] = value;
+	});
+	result.text = text;
+	result.type = "text/html";
+	return [result];
+}
+
+function deserializeTiddlyWikiFile(text,storeAreaEnd,isTiddlyWiki5,fields) {
+	var results = [],
+		endOfDivRegExp = /(<\/div>\s*)/gi,
+		startPos = storeAreaEnd,
+		defaultType = isTiddlyWiki5 ? "application/vnd.tiddlywiki": "application/vnd.tiddlywiki2";
 	endOfDivRegExp.lastIndex = startPos;
 	var match = endOfDivRegExp.exec(text);
-		while(match && startPos < storeAreaPos[1]) {
+	while(match) {
 		var endPos = endOfDivRegExp.lastIndex,
-				tiddlerFields = parseTiddlerDiv(text.substring(startPos,endPos),fields);
+			tiddlerFields = parseTiddlerDiv(text.substring(startPos,endPos),fields,{type: defaultType});
+		if(!tiddlerFields) {
+			break;
+		}
 		if(tiddlerFields.text !== null) {
 			tiddlerFields.text = $tw.utils.htmlDecode(tiddlerFields.text);
 			results.push(tiddlerFields);
@@ -110,8 +133,7 @@ exports["application/vnd.tiddlywiki2"] = function(text,fields) {
 		startPos = endPos;
 		match = endOfDivRegExp.exec(text);
 	}
-	}
 	return results;
-};
+}
 
 })();

core/modules/widgets/import.js

@@ -136,10 +136,6 @@ ImportWidget.prototype.importFiles = function(files) {
 				}
 			}
 		}
-		// Override HTML files so that they're recognised as TiddlyWiki files
-		if(type === "text/html") {
-			type = "application/vnd.tiddlywiki2";
-		}
 		// Figure out if we're reading a binary file
 		var contentTypeInfo = $tw.config.contentTypeInfo[type],
 			isBinary = contentTypeInfo ? contentTypeInfo.encoding === "base64" : false;