Add support for content type parameters and wikitext variants

boot/boot.js

@@ -479,6 +479,31 @@ $tw.utils.registerFileType = function(type,encoding,extension,options) {
 	$tw.config.contentTypeInfo[type] = {encoding: encoding, extension: extension, flags: options.flags || [], deserializerType: options.deserializerType || type};
 };
 
+/*
+Parses a content type of the form "type/subtype; attribute=value", as per https://www.w3.org/Protocols/rfc1341/4_Content-Type.html
+Returns an array of [type/subtype,attribute,value], with the last two entries being optional
+*/
+$tw.utils.parseContentType = function(contentType) {
+	contentType = contentType || "";
+	var parts = contentType.split(";"),
+		results = [(parts[0] || "").trim()];
+	if(parts[1]) {
+		parts = parts[1].split("=");
+		if(parts[0]) {
+			results[1] = (parts[0] || "").trim();
+			results[2] = (parts[1] || "").trim();
+		}
+	}
+	return results;
+};
+
+/*
+Returns the contentTypeInfo for a type
+*/
+$tw.utils.getContentTypeInfo = function(contentType) {
+	return $tw.config.contentTypeInfo[$tw.utils.parseContentType(contentType)[0]] || null;
+};
+
 /*
 Given an extension, always access the $tw.config.fileExtensionInfo
 using a lowercase extension only.
@@ -1463,6 +1488,7 @@ $tw.Wiki.prototype.processSafeMode = function() {
 Extracts tiddlers from a typed block of text, specifying default field values
 */
 $tw.Wiki.prototype.deserializeTiddlers = function(type,text,srcFields,options) {
+	type = $tw.utils.parseContentType(type)[0];
 	srcFields = srcFields || Object.create(null);
 	options = options || {};
 	var deserializer = $tw.Wiki.tiddlerDeserializerModules[options.deserializer],

core/modules/commands/fetch.js

@@ -132,7 +132,7 @@ Command.prototype.processBody = function(body,type,options,url) {
 	// Collect the tiddlers in a wiki
 	var incomingWiki = new $tw.Wiki();
 	if(options.raw) {
-		var typeInfo = type ? $tw.config.contentTypeInfo[type] : null,
+		var typeInfo = type ? $tw.utils.getContentTypeInfo(type) : null,
 			encoding = typeInfo ? typeInfo.encoding : "utf8";
 		incomingWiki.addTiddler(new $tw.Tiddler({
 			title: url,

core/modules/commands/save.js

@@ -37,7 +37,7 @@ Command.prototype.execute = function() {
 	$tw.utils.each(tiddlers,function(title) {
 		var tiddler = self.commander.wiki.getTiddler(title),
 			type = tiddler.fields.type || "text/vnd.tiddlywiki",
-			contentTypeInfo = $tw.config.contentTypeInfo[type] || {encoding: "utf8"},
+			contentTypeInfo = $tw.utils.getContentTypeInfo(type) || {encoding: "utf8"},
 			filepath = path.resolve(self.commander.outputPath,wiki.filterTiddlers(filenameFilter,$tw.rootWidget,wiki.makeTiddlerIterator([title]))[0]);
 		if(self.commander.verbose) {
 			console.log("Saving \"" + title + "\" to \"" + filepath + "\"");

core/modules/commands/savetiddler.js

@@ -35,7 +35,7 @@ Command.prototype.execute = function() {
 		tiddler = this.commander.wiki.getTiddler(title);
 	if(tiddler) {
 		var type = tiddler.fields.type || "text/vnd.tiddlywiki",
-			contentTypeInfo = $tw.config.contentTypeInfo[type] || {encoding: "utf8"};
+			contentTypeInfo = $tw.utils.getContentTypeInfo(type) || {encoding: "utf8"};
 		$tw.utils.createFileDirectories(filename);
 		fs.writeFile(filename,tiddler.fields.text,contentTypeInfo.encoding,function(err) {
 			self.callback(err);

core/modules/commands/savetiddlers.js

@@ -44,7 +44,7 @@ Command.prototype.execute = function() {
 	$tw.utils.each(tiddlers,function(title) {
 		var tiddler = self.commander.wiki.getTiddler(title),
 			type = tiddler.fields.type || "text/vnd.tiddlywiki",
-			contentTypeInfo = $tw.config.contentTypeInfo[type] || {encoding: "utf8"},
+			contentTypeInfo = $tw.utils.getContentTypeInfo(type) || {encoding: "utf8"},
 			filename = path.resolve(pathname,encodeURIComponent(title));
 		fs.writeFileSync(filename,tiddler.fields.text,contentTypeInfo.encoding);
 	});

core/modules/parsers/htmlfragmentparser.js

@@ -0,0 +1,28 @@
+/*\
+title: $:/core/modules/parsers/htmlfragmentparser.js
+type: application/javascript
+module-type: parser
+
+Inherits from the base wikitext parser but is forced into inline mode
+
+\*/
+(function(){
+
+/*jslint node: true, browser: true */
+/*global $tw: false */
+"use strict";
+
+var WikiParser = require("$:/core/modules/parsers/wikiparser/wikiparser.js")["text/vnd.tiddlywiki"];
+
+var PRAGMAS = "\\rules only html entity\n";
+
+var HtmlFragmentWikiParser = function(type,text,options) {
+	var parser = new WikiParser(type,PRAGMAS + text,$tw.utils.extend({},options,{parseAsInline: true}));
+	this.tree = parser.tree;
+	this.prototype = parser.prototype;
+};
+
+exports["text/html+fragment"] = HtmlFragmentWikiParser;
+
+})();
+

core/modules/parsers/wikiparser/wikiparser.js

@@ -48,6 +48,10 @@ var WikiParser = function(type,text,options) {
 	}
 	// Save the parse text
 	this.type = type || "text/vnd.tiddlywiki";
+	this.variant = null;
+	if(options.paramName === "variant" && options.paramValue) {
+		this.variant = options.paramValue;
+	}
 	this.source = text || "";
 	this.sourceLength = this.source.length;
 	// Flag for ignoring whitespace
@@ -59,6 +63,29 @@ var WikiParser = function(type,text,options) {
 	// Instantiate the parser block and inline rules
 	this.blockRules = this.instantiateRules(this.blockRuleClasses,"block",0);
 	this.inlineRules = this.instantiateRules(this.inlineRuleClasses,"inline",0);
+	// Setup the selected variant
+	if(this.variant) {
+		var variantData = this.wiki.getTiddlerDataCached("$:/config/WikiParserVariant/" + this.variant);
+		if(variantData) {
+			// Whitespace setting
+			switch(variantData.whitespace) {
+				case "notrim":
+					this.configTrimWhiteSpace = false;
+					break;
+				case "trim":
+					// Intentional fallthrough
+				default:
+					this.configTrimWhiteSpace = false;
+					break;
+			}
+			// Setup rules
+			if(variantData.rules && variantData.rules.only) {
+				this.amendRules("only",variantData.rules.only);
+			} else if(variantData.rules && variantData.rules.except) {
+				this.amendRules("except",variantData.rules.except);
+			}
+		}
+	}
 	// Parse any pragmas
 	this.tree = [];
 	var topBranch = this.parsePragmas();
@@ -386,22 +413,18 @@ Amend the rules used by this instance of the parser
 WikiParser.prototype.amendRules = function(type,names) {
 	names = names || [];
 	// Define the filter function
-	var keepFilter;
+	var target;
 	if(type === "only") {
-		keepFilter = function(name) {
-			return names.indexOf(name) !== -1;
-		};
+		target = true;
 	} else if(type === "except") {
-		keepFilter = function(name) {
-			return names.indexOf(name) === -1;
-		};
+		target = false;
 	} else {
 		return;
 	}
 	// Define a function to process each of our rule arrays
 	var processRuleArray = function(ruleArray) {
 		for(var t=ruleArray.length-1; t>=0; t--) {
-			if(!keepFilter(ruleArray[t].rule.name)) {
+			if((names.indexOf(ruleArray[t].rule.name) === -1) === target) {
 				ruleArray.splice(t,1);
 			}
 		}

core/modules/utils/filesystem.js

@@ -240,7 +240,7 @@ exports.generateTiddlerFileInfo = function(tiddler,options) {
 		}
 	}
 	// Take the file extension from the tiddler content type
-	var contentTypeInfo = $tw.config.contentTypeInfo[fileInfo.type] || {extension: ""};
+	var contentTypeInfo = $tw.utils.getContentTypeInfo(fileInfo.type) || {extension: ""};
 	// Generate the filepath
 	fileInfo.filepath = $tw.utils.generateTiddlerFilepath(tiddler.fields.title,{
 		extension: contentTypeInfo.extension,
@@ -326,7 +326,7 @@ exports.saveTiddlerToFile = function(tiddler,fileInfo,callback) {
 	$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
 	if(fileInfo.hasMetaFile) {
 		// Save the tiddler as a separate body and meta file
-		var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
+		var typeInfo = $tw.utils.getContentTypeInfo(tiddler.fields.type || "text/plain") || {encoding: "utf8"};
 		fs.writeFile(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding,function(err) {
 			if(err) {
 				return callback(err);
@@ -353,7 +353,7 @@ exports.saveTiddlerToFileSync = function(tiddler,fileInfo) {
 	$tw.utils.createDirectory(path.dirname(fileInfo.filepath));
 	if(fileInfo.hasMetaFile) {
 		// Save the tiddler as a separate body and meta file
-		var typeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/plain"] || {encoding: "utf8"};
+		var typeInfo = $tw.utils.getContentTypeInfo(tiddler.fields.type || "text/plain") || {encoding: "utf8"};
 		fs.writeFileSync(fileInfo.filepath,tiddler.fields.text,typeInfo.encoding);
 		fs.writeFileSync(fileInfo.filepath + ".meta",tiddler.getFieldStringBlock({exclude: ["text","bag"]}),"utf8");
 	} else {

core/modules/utils/utils.js

@@ -760,7 +760,7 @@ Convert text and content type to a data URI
 */
 exports.makeDataUri = function(text,type,_canonical_uri) {
 	type = type || "text/vnd.tiddlywiki";
-	var typeInfo = $tw.config.contentTypeInfo[type] || $tw.config.contentTypeInfo["text/plain"],
+	var typeInfo = $tw.utils.getContentTypeInfo(type) || $tw.utils.getContentTypeInfo("text/plain"),
 		isBase64 = typeInfo.encoding === "base64",
 		parts = [];
 	if(_canonical_uri) {

core/modules/widgets/edit.js

@@ -70,10 +70,10 @@ EditWidget.prototype.getEditorType = function() {
 			type = tiddler.fields.type;
 		}
 	}
-	type = type || "text/vnd.tiddlywiki";
+	type = $tw.utils.parseContentType(type || "text/vnd.tiddlywiki")[0];
 	var editorType = this.wiki.getTiddlerText(EDITOR_MAPPING_PREFIX + type);
 	if(!editorType) {
-		var typeInfo = $tw.config.contentTypeInfo[type];
+		var typeInfo = $tw.utils.getContentTypeInfo(type);
 		if(typeInfo && typeInfo.encoding === "base64") {
 			editorType = "binary";
 		} else {

core/modules/wiki.js

@@ -214,7 +214,7 @@ exports.isTemporaryTiddler = function(title) {
 exports.isImageTiddler = function(title) {
 	var tiddler = this.getTiddler(title);
 	if(tiddler) {		
-		var contentTypeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/vnd.tiddlywiki"];
+		var contentTypeInfo = $tw.utils.getContentTypeInfo(tiddler.fields.type || "text/vnd.tiddlywiki");
 		return !!contentTypeInfo && contentTypeInfo.flags.indexOf("image") !== -1;
 	} else {
 		return null;
@@ -224,7 +224,7 @@ exports.isImageTiddler = function(title) {
 exports.isBinaryTiddler = function(title) {
 	var tiddler = this.getTiddler(title);
 	if(tiddler) {		
-		var contentTypeInfo = $tw.config.contentTypeInfo[tiddler.fields.type || "text/vnd.tiddlywiki"];
+		var contentTypeInfo = $tw.utils.getContentTypeInfo(tiddler.fields.type || "text/vnd.tiddlywiki");
 		return !!contentTypeInfo && contentTypeInfo.encoding === "base64";
 	} else {
 		return null;
@@ -877,14 +877,17 @@ exports.initParsers = function(moduleType) {
 
 /*
 Parse a block of text of a specified MIME type
-	type: content type of text to be parsed
+	contentType: content type of text to be parsed
 	text: text
 	options: see below
 Options include:
 	parseAsInline: if true, the text of the tiddler will be parsed as an inline run
 	_canonical_uri: optional string of the canonical URI of this content
 */
-exports.parseText = function(type,text,options) {
+exports.parseText = function(contentType,text,options) {
+	// Parse the parameteter from the content type
+	var parsedContentType = $tw.utils.parseContentType(contentType),
+		type = parsedContentType[0];
 	text = text || "";
 	options = options || {};
 	// Select a parser
@@ -902,7 +905,9 @@ exports.parseText = function(type,text,options) {
 	return new Parser(type,text,{
 		parseAsInline: options.parseAsInline,
 		wiki: this,
-		_canonical_uri: options._canonical_uri
+		_canonical_uri: options._canonical_uri,
+		paramName: parsedContentType[1],
+		paramValue: parsedContentType[2]
 	});
 };
 
@@ -1181,7 +1186,7 @@ exports.search = function(text,options) {
 		if(!tiddler) {
 			tiddler = new $tw.Tiddler({title: title, text: "", type: "text/vnd.tiddlywiki"});
 		}
-		var contentTypeInfo = $tw.config.contentTypeInfo[tiddler.fields.type] || $tw.config.contentTypeInfo["text/vnd.tiddlywiki"],
+		var contentTypeInfo = $tw.utils.getContentTypeInfo(tiddler.fields.type) || $tw.utils.getContentTypeInfo("text/vnd.tiddlywiki"),
 			searchFields;
 		// Get the list of fields we're searching
 		if(options.excludeField) {
@@ -1335,7 +1340,7 @@ exports.readFile = function(file,options) {
 		}
 	}
 	// Figure out if we're reading a binary file
-	var contentTypeInfo = $tw.config.contentTypeInfo[type],
+	var contentTypeInfo = $tw.utils.getContentTypeInfo(type),
 		isBinary = contentTypeInfo ? contentTypeInfo.encoding === "base64" : false;
 	// Log some debugging information
 	if($tw.log.IMPORT) {

core/wiki/wikiparser-variants/legacy.tid

@@ -0,0 +1,7 @@
+title: $:/config/WikiParserVariant/legacy
+type: application/json
+caption: Legacy
+description: Legacy TiddlyWiki 5 (including CamelCase)
+
+{
+}

core/wiki/wikiparser-variants/standard.tid

@@ -0,0 +1,11 @@
+title: $:/config/WikiParserVariant/standard
+type: application/json
+caption: Standard
+description: Standard modern TiddlyWiki 5 
+
+{
+	"whitespace": "trim",
+	"rules": {
+		"except": "wikilink"
+	}
+}

editions/prerelease/tiddlers/Release 5.1.23.tid

@@ -43,7 +43,7 @@ type: text/vnd.tiddlywiki
 ! Usability Improvements
 
 * [[Added|https://github.com/Jermolene/TiddlyWiki5/commit/8cf458d3b3f1e38c14a2819529e08dca4a7e297c]] "Solarized Dark" palette
-* [[Updated|https://github.com/Jermolene/TiddlyWiki5/pull/4590]] Vanilla theme to use palette colours for the [[browser selection outline|https://developer.mozilla.org/en-US/docs/Web/CSS/::selection]]
+* [[Updated|https://github.com/Jermolene/TiddlyWiki5/pull/4590]] (and [[here|ttps://github.com/Jermolene/TiddlyWiki5/commit/274a07b4fd2ca2d1b95c8ddf52fe055c44260d9b]]) the Vanilla theme to optionally use palette colours for the [[browser selection outline|https://developer.mozilla.org/en-US/docs/Web/CSS/::selection]]
 * [[Added|https://github.com/Jermolene/TiddlyWiki5/commit/1546a4a1895b93a47b79c9d37b94be039604443a]] warning message about using the online plugin library with the client-server configuration
 * [[Improved|https://github.com/Jermolene/TiddlyWiki5/pull/4585]] Gruvbox palette readability of toolbar buttons
 * [[Changed|https://github.com/Jermolene/TiddlyWiki5/commit/9cd5415dfe54b47819920aa3cf6ac2d5e3a9188e]] favicon for the prerelease edition

editions/prerelease/tiddlers/system/TiddlyWiki Pre-release.tid

@@ -1,5 +1,6 @@
 title: TiddlyWiki Pre-release
 modified: 20150428204930183
+type: text/vnd.tiddlywiki; variant=standard
 
 This is a pre-release build of TiddlyWiki, [[also available in empty form|https://tiddlywiki.com/prerelease/empty.html]]. It is provided for testing purposes. ''Please don't try to use it for anything important'' -- you should use the latest official release from https://tiddlywiki.com.
 

plugins/tiddlywiki/aws/modules/command.js

@@ -109,7 +109,7 @@ Command.prototype.subCommands["s3-rendertiddler"] = function() {
 	// Render the tiddler
 	var text = this.commander.wiki.renderTiddler(type,title,{variables: variables}),
 		type = "text/plain",
-		encoding = ($tw.config.contentTypeInfo[type] || {encoding: "utf8"}).encoding;
+		encoding = ($tw.utils.getContentTypeInfo(type) || {encoding: "utf8"}).encoding;
 	// Zip it if needed
 	if(zipfilename) {
 		var JSZip = require("$:/plugins/tiddlywiki/jszip/jszip.js"),
@@ -177,7 +177,7 @@ Command.prototype.subCommands["s3-savetiddler"] = function() {
 		tiddler = wiki.getTiddler(title),
 		text = tiddler.fields.text,
 		type = tiddler.fields.type,
-		encoding = ($tw.config.contentTypeInfo[type] || {encoding: "utf8"}).encoding;
+		encoding = ($tw.utils.getContentTypeInfo(type) || {encoding: "utf8"}).encoding;
 	// Check parameters
 	if(!title || !region || !bucket || !filename) {
 		throw "Missing parameters";

plugins/tiddlywiki/aws/modules/utils.js

@@ -55,7 +55,7 @@ function putFile(region,bucketName,title,text,type,callback) {
 		s3bucket = new AWS.S3({
 			region: region
 		}),
-		encoding = ($tw.config.contentTypeInfo[type] || {encoding: "utf8"}).encoding,
+		encoding = ($tw.utils.getContentTypeInfo(type) || {encoding: "utf8"}).encoding,
 		params = {
 			Bucket: bucketName,
 			Key: title,

plugins/tiddlywiki/classictools/modules/recipe.js

@@ -39,7 +39,7 @@ exports["text/vnd.tiddlywiki2-recipe"] = function(text,fields) {
 		loadTiddlersFromFile = function(sourcePath,prefix) {
 			var ext = path.extname(sourcePath),
 				extensionInfo = $tw.utils.getFileExtensionInfo(ext),
-				typeInfo = extensionInfo ? $tw.config.contentTypeInfo[extensionInfo.type] : null,
+				typeInfo = extensionInfo ? $tw.utils.getContentTypeInfo(extensionInfo.type) : null,
 				data = fs.readFileSync(sourcePath,typeInfo ? typeInfo.encoding : "utf8"),
 				fields = {title: sourcePath},
 				tids = self.deserializeTiddlers(ext,data,fields),