Extend encodeuricomponent to process additional characters (#7128)

* First commit

* Fix version number in docs

* Add code comment

core/modules/commands/rendertiddlers.js

@@ -57,7 +57,7 @@ Command.prototype.execute = function() {
 				exportPath = path.resolve(outputPath,macroPath + extension);
 			}
 		}
-		var finalPath = exportPath || path.resolve(pathname,encodeURIComponent(title) + extension);
+		var finalPath = exportPath || path.resolve(pathname,$tw.utils.encodeURIComponentExtended(title) + extension);
 		$tw.utils.createFileDirectories(finalPath);
 		fs.writeFileSync(finalPath,text,"utf8");
 	});

core/modules/commands/savelibrarytiddlers.js

@@ -65,7 +65,7 @@ Command.prototype.execute = function() {
 	$tw.utils.each(filteredPluginList,function(title) {
 		var tiddler = containerData.tiddlers[title];
 		// Save each JSON file and collect the skinny data
-		var pathname = path.resolve(self.commander.outputPath,basepath + encodeURIComponent(title) + ".json");
+		var pathname = path.resolve(self.commander.outputPath,basepath + $tw.utils.encodeURIComponentExtended(title) + ".json");
 		$tw.utils.createFileDirectories(pathname);
 		fs.writeFileSync(pathname,JSON.stringify(tiddler),"utf8");
 		// Collect the skinny list data

core/modules/commands/savetiddlers.js

@@ -45,7 +45,7 @@ Command.prototype.execute = function() {
 		var tiddler = self.commander.wiki.getTiddler(title),
 			type = tiddler.fields.type || "text/vnd.tiddlywiki",
 			contentTypeInfo = $tw.config.contentTypeInfo[type] || {encoding: "utf8"},
-			filename = path.resolve(pathname,encodeURIComponent(title));
+			filename = path.resolve(pathname,$tw.utils.encodeURIComponentExtended(title));
 		fs.writeFileSync(filename,tiddler.fields.text,contentTypeInfo.encoding);
 	});
 	return null;

core/modules/filters/encodings.js

@@ -27,7 +27,7 @@ exports.decodeuricomponent = function(source,operator,options) {
 exports.encodeuricomponent = function(source,operator,options) {
 	var results = [];
 	source(function(tiddler,title) {
-		results.push(encodeURIComponent(title));
+		results.push($tw.utils.encodeURIComponentExtended(title));
 	});
 	return results;
 };

core/modules/utils/filesystem.js

@@ -393,7 +393,7 @@ exports.generateTiddlerFilepath = function(title,options) {
 	} while(fs.existsSync(fullPath));
 	// If the last write failed with an error, or if path does not start with:
 	//	the resolved options.directory, the resolved wikiPath directory, the wikiTiddlersPath directory, 
-	//	or the 'originalpath' directory, then encodeURIComponent() and resolve to tiddler directory.
+	//	or the 'originalpath' directory, then $tw.utils.encodeURIComponentExtended() and resolve to tiddler directory.
 	var writePath = $tw.hooks.invokeHook("th-make-tiddler-path",fullPath,fullPath),
 		encode = (options.fileInfo || {writeError: false}).writeError == true;
 	if(!encode) {
@@ -403,7 +403,7 @@ exports.generateTiddlerFilepath = function(title,options) {
 			writePath.indexOf(path.resolve($tw.boot.wikiTiddlersPath,originalpath)) == 0 );
 		}
 	if(encode) {
-		writePath = path.resolve(directory,encodeURIComponent(fullPath));
+		writePath = path.resolve(directory,$tw.utils.encodeURIComponentExtended(fullPath));
 	}
 	// Return the full path to the file
 	return writePath;

core/modules/utils/utils.js

@@ -685,6 +685,16 @@ exports.escapeRegExp = function(s) {
     return s.replace(/[\-\/\\\^\$\*\+\?\.\(\)\|\[\]\{\}]/g, '\\$&');
 };
 
+/*
+Extended version of encodeURIComponent that encodes additional characters including
+those that are illegal within filepaths on various platforms including Windows
+*/
+exports.encodeURIComponentExtended = function(s) {
+	return encodeURIComponent(s).replace(/[!'()*]/g,function(c) {
+		return "%" + c.charCodeAt(0).toString(16).toUpperCase();
+	});
+};
+
 // Checks whether a link target is external, i.e. not a tiddler title
 exports.isLinkExternal = function(to) {
 	var externalRegExp = /^(?:file|http|https|mailto|ftp|irc|news|obsidian|data|skype):[^\s<>{}\[\]`|"\\^]+(?:\/|\b)/i;

core/modules/widgets/link.js

@@ -97,8 +97,8 @@ LinkWidget.prototype.renderLink = function(parent,nextSibling) {
 		// Expand the tv-wikilink-template variable to construct the href
 		var wikiLinkTemplateMacro = this.getVariable("tv-wikilink-template"),
 			wikiLinkTemplate = wikiLinkTemplateMacro ? wikiLinkTemplateMacro.trim() : "#$uri_encoded$";
-		wikiLinkText = $tw.utils.replaceString(wikiLinkTemplate,"$uri_encoded$",encodeURIComponent(this.to));
+		wikiLinkText = $tw.utils.replaceString(wikiLinkTemplate,"$uri_encoded$",$tw.utils.encodeURIComponentExtended(this.to));
-		wikiLinkText = $tw.utils.replaceString(wikiLinkText,"$uri_doubleencoded$",encodeURIComponent(encodeURIComponent(this.to)));
+		wikiLinkText = $tw.utils.replaceString(wikiLinkText,"$uri_doubleencoded$",$tw.utils.encodeURIComponentExtended($tw.utils.encodeURIComponentExtended(this.to)));
 	}
 	// Override with the value of tv-get-export-link if defined
 	wikiLinkText = this.getVariable("tv-get-export-link",{params: [{name: "to",value: this.to}],defaultValue: wikiLinkText});

core/modules/widgets/view.js

@@ -168,11 +168,11 @@ ViewWidget.prototype.getValueAsHtmlTextEncoded = function() {
 };
 
 ViewWidget.prototype.getValueAsUrlEncoded = function() {
-	return encodeURIComponent(this.getValueAsText());
+	return $tw.utils.encodeURIComponentExtended(this.getValueAsText());
 };
 
 ViewWidget.prototype.getValueAsDoubleUrlEncoded = function() {
-	return encodeURIComponent(encodeURIComponent(this.getValueAsText()));
+	return $tw.utils.encodeURIComponentExtended($tw.utils.encodeURIComponentExtended(this.getValueAsText()));
 };
 
 ViewWidget.prototype.getValueAsDate = function(format) {

editions/test/tiddlers/tests/test-filters.js

@@ -1108,6 +1108,10 @@ Tests the filtering mechanism.
 		  );
 		});
 		
+		it("should handle the encodeuricomponent and decodeuricomponent operators", function() {
+			expect(wiki.filterTiddlers("[[<>:\"/\\|?*]encodeuricomponent[]]").join(",")).toBe("%3C%3E%3A%22%2F%5C%7C%3F%2A");
+		});
+	
 	}
 	
 	});

editions/test/tiddlers/tests/test-wikitext.js

@@ -43,7 +43,7 @@ describe("WikiText tests", function() {
 		expect(wiki.renderTiddler("text/html","TiddlerThree")).toBe("<p>The speed of sound</p><p>The light of speed</p>");
 	});
 	it("should support attributes specified as macro invocations", function() {
-		expect(wiki.renderTiddler("text/html","TiddlerFour")).toBe("<p><a class=\"tc-tiddlylink tc-tiddlylink-missing\" href=\"#This%20is%20my%20''amazingly''%20groovy%20macro!\">This is a link</a></p>");
+		expect(wiki.renderTiddler("text/html","TiddlerFour")).toBe("<p><a class=\"tc-tiddlylink tc-tiddlylink-missing\" href=\"#This%20is%20my%20%27%27amazingly%27%27%20groovy%20macro%21\">This is a link</a></p>");
 	});
 	it("should identify wikiwords to automatically link", function() {
 		expect(wiki.renderText("text/html","text/vnd-tiddlywiki","No wikilinks here").indexOf("<a") !== -1).toBe(false);

editions/tw5.com/tiddlers/filters/encodeuricomponent_Operator.tid

@@ -1,6 +1,6 @@
 caption: encodeuricomponent
 created: 20161017152747386
-modified: 20161017152809900
+modified: 20230119174350062
 op-input: a [[selection of titles|Title Selection]]
 op-output: the input with URI component encoding applied
 op-parameter: 
@@ -13,4 +13,6 @@ from-version: 5.1.14
 
 See Mozilla Developer Network for details of the [[encodeURIComponent|https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent]] operation.
 
+<<.from-version "5.2.6">> In addition to the characters mentioned in the article above, the following additional characters are also percent encoded: `!'()*`
+
 <<.operator-examples "encodeuricomponent">>