Simplify Permalink/Permaview URLs (#7729)

* Simplify Permalink/Permaview URLs

* Fix lint warnings by removing arrow functions

* Remove commented sample code

* Remove post-ES5 code

* Add many more allowable non-percent-encodedcharacters

* Fix more ES6+ stuff, add end-of-sentence padding character.

* Fix to match standards

* Move the new code from boot to util

* Change from custom map/filter to $tw.utils.each

* Make `each` blocks multi-line

* Move the permalink handling to its own file

* Remove auto-navigation

* Revert "Remove auto-navigation"

This reverts commit ca1e5cf387.

core/modules/startup/story.js

@@ -122,10 +122,10 @@ function openStartupTiddlers(options) {
 		var hash = $tw.locationHash.substr(1),
 			split = hash.indexOf(":");
 		if(split === -1) {
-			target = $tw.utils.decodeURIComponentSafe(hash.trim());
+			target = $tw.utils.decodeTWURITarget(hash.trim());
 		} else {
-			target = $tw.utils.decodeURIComponentSafe(hash.substr(0,split).trim());
+			target = $tw.utils.decodeTWURITarget(hash.substr(0,split).trim());
-			storyFilter = $tw.utils.decodeURIComponentSafe(hash.substr(split + 1).trim());
+			storyFilter = $tw.utils.decodeTWURIList(hash.substr(split + 1).trim());
 		}
 	}
 	// If the story wasn't specified use the current tiddlers or a blank story
@@ -198,19 +198,19 @@ function updateLocationHash(options) {
 	// Assemble the location hash
 	switch(options.updateAddressBar) {
 		case "permalink":
-			$tw.locationHash = "#" + encodeURIComponent(targetTiddler);
+			$tw.locationHash = "#" + $tw.utils.encodeTiddlerTitle(targetTiddler);
 			break;
 		case "permaview":
-			$tw.locationHash = "#" + encodeURIComponent(targetTiddler) + ":" + encodeURIComponent($tw.utils.stringifyList(storyList));
+			$tw.locationHash = "#" + $tw.utils.encodeTiddlerTitle(targetTiddler) + ":" + $tw.utils.encodeFilterPath($tw.utils.stringifyList(storyList));
 			break;
 	}
 	// Copy URL to the clipboard
 	switch(options.copyToClipboard) {
 		case "permalink":
-			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler));
+			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + $tw.utils.encodeTiddlerTitle(targetTiddler));
 			break;
 		case "permaview":
-			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + encodeURIComponent(targetTiddler) + ":" + encodeURIComponent($tw.utils.stringifyList(storyList)));
+			$tw.utils.copyToClipboard($tw.utils.getLocationPath() + "#" + $tw.utils.encodeTiddlerTitle(targetTiddler) + ":" + $tw.utils.encodeFilterPath($tw.utils.stringifyList(storyList)));
 			break;
 	}
 	// Only change the location hash if we must, thus avoiding unnecessary onhashchange events

core/modules/utils/twuri-encoding.js

@@ -0,0 +1,128 @@
+/*\
+title: $:/core/modules/utils/twuri-encoding.js
+type: application/javascript
+module-type: utils
+
+Utility functions related to permalink/permaview encoding/decoding.
+
+\*/
+(function(){
+
+	// The character that will substitute for a space in the URL
+	var SPACE_SUBSTITUTE = "_";
+
+	// The character added to the end to avoid ending with `.`, `?`, `!` or the like
+	var TRAILER = "_";
+
+	// The character that will separate out the list elements in the URL
+	var CONJUNCTION = ";";
+
+	// Those of the allowed url characters claimed by TW
+	var CLAIMED = [SPACE_SUBSTITUTE, ":", CONJUNCTION];
+
+	// Non-alphanumeric characters allowed in a URL fragment
+	// More information at https://www.rfc-editor.org/rfc/rfc3986#appendix-A
+	var VALID_IN_URL_FRAGMENT = "-._~!$&'()*+,;=:@/?".split("");
+
+	// The subset of the pchars we will not percent-encode in permalinks/permaviews
+	var SUBSTITUTES = []
+	$tw.utils.each(VALID_IN_URL_FRAGMENT, function(c) {
+		if (CLAIMED.indexOf(c) === -1) {
+			SUBSTITUTES.push(c)
+		}
+	});
+
+	// A regex to match the percent-encoded characters we will want to replace.
+	// Something similar to the following, depending on SPACE and CONJUNCTION
+	//     /(%2D|%2E|%7E|%21|%24|%26|%27|%28|%29|%2A|%2B|%3B|%3D|%40|%2F|%3F)/g
+
+	var CHAR_MATCH_STR = []
+	$tw.utils.each(SUBSTITUTES, function(c) {
+		CHAR_MATCH_STR.push("%" + c.charCodeAt(0).toString(16).toUpperCase())
+	})
+	var CHAR_MATCH = new RegExp("(" + CHAR_MATCH_STR.join("|") + ")", "g");
+
+	// A regex to match the SPACE_SUBSTITUTE character
+	var SPACE_MATCH = new RegExp("(\\" + SPACE_SUBSTITUTE + ")", "g");
+
+	// A regex to match URLs ending with sentence-ending punctuation
+	var SENTENCE_ENDING = new RegExp("(\\.|\\!|\\?|\\" + TRAILER + ")$", "g");
+
+	// A regex to match URLs ending with sentence-ending punctuation plus the TRAILER
+	var SENTENCE_TRAILING = new RegExp("(\\.|\\!|\\?|\\" + TRAILER + ")\\" + TRAILER + "$", "g");
+
+	// An object mapping the percent encodings back to their source characters
+	var PCT_CHAR_MAP = SUBSTITUTES.reduce(function (a, c) {
+		a["%" + c.charCodeAt(0).toString(16).toUpperCase()] = c
+		return a
+	}, {});
+
+	// Convert a URI List Component encoded string (with the `SPACE_SUBSTITUTE`
+	// value as an allowed replacement for the space character) to a string
+	exports.decodeTWURIList = function(s) {
+		var parts = s.replace(SENTENCE_TRAILING, "$1").split(CONJUNCTION);
+		var withSpaces = []
+		$tw.utils.each(parts, function(s) {
+			withSpaces.push(s.replace(SPACE_MATCH, " "))
+		});
+		var withBrackets  = []
+		$tw.utils.each(withSpaces, function(s) {
+			withBrackets .push(s.indexOf(" ") >= 0 ? "[[" + s + "]]" : s)
+		});
+		return $tw.utils.decodeURIComponentSafe(withBrackets.join(" "));
+	};
+
+	// Convert a URI Target Component encoded string (with the `SPACE_SUBSTITUTE` 
+	// value as an allowed replacement for the space character) to a string
+	exports.decodeTWURITarget = function(s) {
+		return $tw.utils.decodeURIComponentSafe(
+			s.replace(SENTENCE_TRAILING, "$1").replace(SPACE_MATCH, " ")
+		)
+	};
+
+	// Convert a URIComponent encoded title string (with the `SPACE_SUBSTITUTE`
+	// value as an allowed replacement for the space character) to a string
+	exports.encodeTiddlerTitle = function(s) {
+		var extended = s.replace(SENTENCE_ENDING, "$1" + TRAILER)
+		var encoded = encodeURIComponent(extended);
+		var substituted = encoded.replace(/\%20/g, SPACE_SUBSTITUTE);
+		return substituted.replace(CHAR_MATCH, function(_, c) {
+			return PCT_CHAR_MAP[c];
+		});
+	};
+
+	// Convert a URIComponent encoded filter string (with the `SPACE_SUBSTITUTE`
+	// value as an allowed replacement for the space character) to a string
+	exports.encodeFilterPath = function(s) {
+		var parts = s.replace(SENTENCE_ENDING, "$1" + TRAILER)
+			.replace(/\[\[(.+?)\]\]/g, function (_, t) {return t.replace(/ /g, SPACE_SUBSTITUTE )})
+			.split(" ");
+		var nonEmptyParts = []
+		$tw.utils.each(parts, function(p) {
+			if (p) {
+				nonEmptyParts.push (p)
+			}
+		});
+		var trimmed = [];
+		$tw.utils.each(nonEmptyParts, function(s) {
+			trimmed.push(s.trim())
+		});
+		var encoded = [];
+		$tw.utils.each(trimmed, function(s) {
+			encoded.push(encodeURIComponent(s))
+		});
+		var substituted = [];
+		$tw.utils.each(encoded, function(s) {
+			substituted.push(s.replace(/\%20/g, SPACE_SUBSTITUTE))
+		});
+		var replaced = []
+		$tw.utils.each(substituted, function(s) {
+			replaced.push(s.replace(CHAR_MATCH, function(_, c) {
+				return PCT_CHAR_MAP[c];
+			}))
+		});
+		return replaced.join(CONJUNCTION);
+	};
+	
+})();
+	

editions/tw5.com/tiddlers/concepts/PermaLinks.tid

@@ -40,6 +40,22 @@ There are technical restrictions on the legal characters in an URL fragment. To
 
 Both the target tiddler title and the story filter should be URL encoded (but not the separating colon). TiddlyWiki generates properly encoded URLs which can look quite ugly. However, in practice browsers will usually perfectly happily process arbitrary characters in URL fragments. Thus when creating permalinks manually you can choose to ignore URL encoding.
 
+!! Simpler URLS
+
+<<.from-version "5.3.2">> The URLs generated are simplified from the hard-to-read percent encoding when feasible.  Spaces are replaced with underscores (`_`), many punctuation characters are allowed to remain unencoded, and permaview filters receive a simpler encoding.  For example the tiddler "Hard Linebreaks with CSS - Example", which percent-encoded would look like 
+
+> @@font-family:monospace;#Hard%20Linebreaks%20with%20CSS%20-%20Example@@
+
+instead looks like 
+
+> @@font-family:monospace;#Hard_Linebreaks_with_CSS_-_Example@@
+
+Existing story filter URLs like
+
+> @@font-family:monospace;#:[tag[Features]]%20+[limit[5]]@@
+
+will continue to work.
+
 ! Permalink Behaviour
 
 Two important aspects of TiddlyWiki's behaviour with permalinks can be controlled via options in the [[control panel|$:/ControlPanel]] <<.icon $:/core/images/options-button>> ''Settings'' tab: