Replace Markdown parsing library with Remarkable (#3876)

* Replace Markdown parsing library with Remarkable

* Fix handling of block-level elements

* Update documentation

* Add config options for Parser actions

* Add Config options for Remarkable library

* Match code style

* Update documentation

* Handle ordered lists and horizontal rules

* Update to v2.0.0 of Remarkable library

editions/markdowndemo/tiddlers/HelloThere.tid

@@ -1,6 +1,6 @@
 title: HelloThere
 
-This is a demo of TiddlyWiki5 incorporating a plugin for the [[markdown-js|https://github.com/evilstreak/markdown-js]] Markdown parser from Dominic Baggott. The MarkdownExample tiddler below is written in Markdown.
+This is a demo of TiddlyWiki5 incorporating a plugin for parsing tiddlers written in the Markdown language. The plugin uses the [[Remarkable|https://github.com/jonschlinkert/remarkable]] Markdown parser internally. The MarkdownExample tiddler below is written in Markdown.
 
 ! Installation
 
@@ -8,31 +8,4 @@ To add the plugin to your own TiddlyWiki5, just drag this link to the browser wi
 
 [[$:/plugins/tiddlywiki/markdown]]
 
-! Markdown Dialects
-
-By default the markdown parser recognises the original dialect of Markdown [[as described by John Gruber|http://daringfireball.net/projects/markdown/]]. An extended dialect called "Maruku" is also included that provides table support and other advanced features. The syntax extensions are modelled on those of [[PHP Markdown Extra|https://michelf.ca/projects/php-markdown/extra/]].
-
-The configuration tiddler [[$:/config/markdown/dialect]] determines which dialect is used:
-
-|!Dialect |!Description |
-|Gruber |Standard Markdown |
-|Maruku |Extended Maruku Markdown |
-
-
-! Creating ~WikiLinks
-
-Create wiki links with the usual Markdown link syntax targeting `#` and the target tiddler title:
-
-```
-[link text](#TiddlerTitle)
-```
-
-! Images
-
-Markdown image syntax can be used to reference images by tiddler title or an external URI. For example:
-
-```
-![alt text](/path/to/img.jpg "Title")
-
-![alt text](Motovun Jack.jpg "Title")
-```
+{{$:/plugins/tiddlywiki/markdown/usage}}

editions/markdowndemo/tiddlers/MarkdownExample.tid

@@ -19,11 +19,10 @@ web application that allows you type your own Markdown-formatted text
 and translate it to XHTML.
 
 **Note:** This document is itself written using Markdown; you
-can [see the source for it by adding '.text' to the URL] [src].
+can see the source for it by editing this tiddler.
 
-  [s]: /projects/markdown/syntax  "Markdown Syntax"
-  [d]: /projects/markdown/dingus  "Markdown Dingus"
-  [src]: /projects/markdown/basics.text
+  [s]: https://daringfireball.net/projects/markdown/syntax  "Markdown Syntax"
+  [d]: https://daringfireball.net/projects/markdown/dingus  "Markdown Dingus"
 
 
 ## Paragraphs, Headers, Blockquotes ##

plugins/tiddlywiki/markdown/config_breaks.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/breaks
+
+false

plugins/tiddlywiki/markdown/config_linkNewWindow.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/linkNewWindow
+
+true

plugins/tiddlywiki/markdown/config_linkify.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/linkify
+
+false

plugins/tiddlywiki/markdown/config_quotes.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/quotes
+
+“”‘’

plugins/tiddlywiki/markdown/config_renderWikiText.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/renderWikiText
+
+true

plugins/tiddlywiki/markdown/config_renderWikiTextPragma.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/renderWikiTextPragma
+
+\rules only html image macrocallinline syslink transcludeinline wikilink filteredtranscludeblock macrocallblock transcludeblock

plugins/tiddlywiki/markdown/config_typographer.tid

@@ -0,0 +1,3 @@
+title: $:/config/markdown/typographer
+
+false

plugins/tiddlywiki/markdown/dialect.tid

@@ -1,3 +0,0 @@
-title: $:/config/markdown/dialect
-
-Gruber

plugins/tiddlywiki/markdown/files/tiddlywiki.files

@@ -1,10 +1,10 @@
 {
 	"tiddlers": [
 		{
-			"file": "markdown.js",
+			"file": "remarkable.js",
 			"fields": {
 				"type": "application/javascript",
-				"title": "$:/plugins/tiddlywiki/markdown/markdown.js",
+				"title": "$:/plugins/tiddlywiki/markdown/remarkable.js",
 				"module-type": "library"
 			}
 		}

plugins/tiddlywiki/markdown/readme.tid

@@ -1,6 +1,6 @@
 title: $:/plugins/tiddlywiki/markdown/readme
 
-This is a TiddlyWiki plugin for parsing Markdown text, based on the [[markdown-js|https://github.com/evilstreak/markdown-js]] project from Dominic Baggott. 
+This is a TiddlyWiki plugin for parsing Markdown text, using the [[Remarkable|https://github.com/jonschlinkert/remarkable]] library.
 
 It is completely self-contained, and doesn't need an Internet connection in order to work. It works both in the browser and under Node.js.
 

plugins/tiddlywiki/markdown/usage.tid

@@ -1,15 +1,15 @@
 title: $:/plugins/tiddlywiki/markdown/usage
 
-! Markdown Dialects
-
-By default the markdown parser recognises the original dialect of Markdown [[as described by John Gruber|http://daringfireball.net/projects/markdown/]]. An extended dialect called "Maruku" is also included that provides table support and other advanced features. The syntax extensions are modelled on those of [[PHP Markdown Extra|https://michelf.ca/projects/php-markdown/extra/]].
-
-The configuration tiddler [[$:/config/markdown/dialect]] determines which dialect is used:
-
-|!Dialect |!Description |
-|Gruber |Standard Markdown |
-|Maruku |Extended Maruku Markdown |
+! Plugin Configuration
 
+|!Config |!Default |!Description |
+| <code>[[breaks|$:/config/markdown/breaks]]</code>| ``false``|Remarkable library config: Convert '\n' in paragraphs into ``<br>`` |
+| <code>[[linkify|$:/config/markdown/linkify]]</code>| ``false``|Remarkable library config: Autoconvert URL-like text to links |
+| <code>[[linkNewWindow|$:/config/markdown/linkNewWindow]]</code>| ``true``|For external links, should clicking on them open a new window/tab automatically? |
+| <code>[[quotes|$:/config/markdown/quotes]]</code>| ``“”‘’``|Remarkable library config: Double + single quotes replacement pairs, when ``typographer`` enabled |
+| <code>[[renderWikiText|$:/config/markdown/renderWikiText]]</code>| ``true``|After Markdown is parsed, should any text elements be handed off to the ~WikiText parser for further processing? |
+| <code>[[renderWikiTextPragma|$:/config/markdown/renderWikiTextPragma]]</code>| ``\rules only html image macrocallinline syslink transcludeinline wikilink filteredtranscludeblock macrocallblock transcludeblock``|When handing off to the ~WikiText parser, what pragma rules should it follow? |
+| <code>[[typographer|$:/config/markdown/typographer]]</code>| ``false``|Remarkable library config: Enable some language-neutral replacement + quotes beautification |
 
 ! Creating ~WikiLinks
 
@@ -19,6 +19,12 @@ Create wiki links with the usual Markdown link syntax targeting `#` and the targ
 [link text](#TiddlerTitle)
 ```
 
+If the target tiddler has a space in its name, that name must be URL-escaped to be detected as a URL:
+
+```
+[link text](#Test%20Tiddler)
+```
+
 ! Images
 
 Markdown image syntax can be used to reference images by tiddler title or an external URI. For example:

plugins/tiddlywiki/markdown/wrapper.js

@@ -3,7 +3,7 @@ title: $:/plugins/tiddlywiki/markdown/wrapper.js
 type: application/javascript
 module-type: parser
 
-Wraps up the markdown-js parser for use in TiddlyWiki5
+Wraps up the remarkable parser for use as a Parser in TiddlyWiki
 
 \*/
 (function(){
@@ -12,78 +12,211 @@ Wraps up the markdown-js parser for use in TiddlyWiki5
 /*global $tw: false */
 "use strict";
 
-var markdown = require("$:/plugins/tiddlywiki/markdown/markdown.js");
+const { Remarkable, linkify, utils } = require("$:/plugins/tiddlywiki/markdown/remarkable.js");
 
-var CONFIG_DIALECT_TIDDLER = "$:/config/markdown/dialect",
-	DEFAULT_DIALECT = "Gruber";
-
-function transformNodes(nodes) {
-	var results = [];
-	for(var index=0; index<nodes.length; index++) {
-		results.push(transformNode(nodes[index]));
+///// Set up configuration options /////
+function parseAsBoolean(tiddlerName) {
+	return $tw.wiki.getTiddlerText(tiddlerName).toLowerCase() === "true";
 }
-	return results;
+var pluginOpts = {
+	linkNewWindow: parseAsBoolean("$:/config/markdown/linkNewWindow"),
+	renderWikiText: parseAsBoolean("$:/config/markdown/renderWikiText"),
+	renderWikiTextPragma: $tw.wiki.getTiddlerText("$:/config/markdown/renderWikiTextPragma").trim()
+};
+var remarkableOpts = {
+	breaks: parseAsBoolean("$:/config/markdown/breaks"),
+	quotes: $tw.wiki.getTiddlerText("$:/config/markdown/quotes"),
+	typographer: parseAsBoolean("$:/config/markdown/typographer")
+};
+
+var md = new Remarkable(remarkableOpts);
+
+if (parseAsBoolean("$:/config/markdown/linkify")) {
+	md = md.use(linkify);
 }
 
-function transformNode(node) {
-	if($tw.utils.isArray(node)) {
-		var p = 0,
-			widget = {type: "element", tag: node[p++]};
-		if(!$tw.utils.isArray(node[p]) && typeof(node[p]) === "object") {
-			widget.attributes = {};
-			$tw.utils.each(node[p++],function(value,name) {
-				widget.attributes[name] = {type: "string", value: value};
+function findTagWithType(nodes, startPoint, type, level) {
+	for (var i = startPoint; i < nodes.length; i++) {
+		if (nodes[i].type === type && nodes[i].level === level) {
+			return i;
+		}
+	}
+	return false;
+}
+
+/**
+ * Remarkable creates nodes that look like:
+ * [
+ *   { type: 'paragraph_open'},
+ *   { type: 'inline', content: 'Hello World', children:[{type: 'text', content: 'Hello World'}]},
+ *   { type: 'paragraph_close'}
+ * ]
+ *
+ * But TiddlyWiki wants the Parser (https://tiddlywiki.com/dev/static/Parser.html) to emit nodes like:
+ *
+ * [
+ *   { type: 'element', tag: 'p', children: [{type: 'text', text: 'Hello World'}]}
+ * ]
+ */
+function convertNodes(remarkableTree, isStartOfInline) {
+	let out = [];
+
+	function wrappedElement(elementTag, currentIndex, currentLevel, closingType, nodes) {
+		var j = findTagWithType(nodes, currentIndex + 1, closingType, currentLevel);
+		if (j === false) {
+			console.error("Failed to find a " + closingType + " node after position " + currentIndex);
+			console.log(nodes);
+			return currentIndex + 1;
+		}
+		let children = convertNodes(nodes.slice(currentIndex + 1, j));
+
+		out.push({
+			type: "element",
+			tag: elementTag,
+			children: children
+		});
+		return j;
+	}
+
+	for (var i = 0; i < remarkableTree.length; i++) {
+		var currentNode = remarkableTree[i];
+		if (currentNode.type === "paragraph_open") {
+			i = wrappedElement("p", i, currentNode.level, "paragraph_close", remarkableTree);
+		} else if (currentNode.type === "heading_open") {
+			i = wrappedElement("h" + currentNode.hLevel, i, currentNode.level, "heading_close", remarkableTree);
+		} else if (currentNode.type === "bullet_list_open") {
+			i = wrappedElement("ul", i, currentNode.level, "bullet_list_close", remarkableTree);
+		} else if (currentNode.type == 'ordered_list_open') {
+			i = wrappedElement('ol', i, currentNode.level,'ordered_list_close', remarkableTree);
+		} else if (currentNode.type === "list_item_open") {
+			i = wrappedElement("li", i, currentNode.level, "list_item_close", remarkableTree);
+		} else if (currentNode.type === "link_open") {
+			var j = findTagWithType(remarkableTree, i + 1, "link_close", currentNode.level);
+
+			if (currentNode.href[0] !== "#") {
+				// External link
+				var attributes = {
+					href: { type: "string", value: currentNode.href }
+				};
+				if (pluginOpts.linkNewWindow) {
+					attributes.target = { type: "string", value: "_blank" };
+				}
+				out.push({
+					type: "element",
+					tag: "a",
+					attributes: attributes,
+					children: convertNodes(remarkableTree.slice(i + 1, j))
+				});
+			} else {
+				// Internal link
+				out.push({
+					type: "link",
+					attributes: {
+						to: { type: "string", value: decodeURI(currentNode.href.substr(1)) }
+					},
+					children: convertNodes(remarkableTree.slice(i + 1, j))
 				});
 			}
-		widget.children = transformNodes(node.slice(p++));
-		// Massage images into the image widget
-		if(widget.tag === "img") {
-			widget.type = "image";
-			if(widget.attributes.alt) {
-				widget.attributes.tooltip = widget.attributes.alt;
-				delete widget.attributes.alt;
+			i = j;
+		} else if (currentNode.type.substr(currentNode.type.length - 5) === "_open") {
+			var tagName = currentNode.type.substr(0, currentNode.type.length - 5);
+			i = wrappedElement(tagName, i, currentNode.level, tagName + "_close", remarkableTree);
+		} else if (currentNode.type === "code") {
+			out.push({
+				type: "element",
+				tag: currentNode.block ? "pre" : "code",
+				children: [{ type: "text", text: currentNode.content }]
+			});
+		} else if (currentNode.type === "fence") {
+			out.push({
+				type: "codeblock",
+				attributes: {
+					language: { type: "string", value: currentNode.params },
+					code: { type: "string", value: currentNode.content }
 				}
-			if(widget.attributes.src) {
-				widget.attributes.source = widget.attributes.src;
-				delete widget.attributes.src;
+			});
+		} else if (currentNode.type === "image") {
+			out.push({
+				type: "image",
+				attributes: {
+					tooltip: { type: "string", value: currentNode.alt },
+					source: { type: "string", value: currentNode.src }
 				}
-		}
-		// Convert internal links to proper wikilinks
-		if (widget.tag === "a" && widget.attributes.href.value[0] === "#") {
-			widget.type = "link";
-			widget.attributes.to = widget.attributes.href;
-			if (widget.attributes.to.type === "string") {
-				//Remove '#' before conversion to wikilink
-				widget.attributes.to.value = widget.attributes.to.value.substr(1);
-			}
-			//Children is fine
-			delete widget.tag;
-			delete widget.attributes.href;
-		}
-		return widget;
+			});
+		} else if (currentNode.type === "softbreak") {
+			out.push({
+				type: "element",
+				tag: "br",
+			});
+		} else if (currentNode.type == 'hr') {
+			out.push({
+				type: 'element',
+				tag: 'hr',
+			});
+		} else if (currentNode.type === "inline") {
+			out = out.concat(convertNodes(currentNode.children, true));
+		} else if (currentNode.type === "text") {
+			if (!pluginOpts.renderWikiText) {
+				out.push({
+					type: "text",
+					text: currentNode.content
+				});
 			} else {
-		return {type: "text", text: node};
+				// The Markdown compiler thinks this is just text.
+				// Hand off to the WikiText parser to see if there's more to render
+
+				// If we're inside a block element (div, p, td, h1), and this is the first child in the tree,
+				// handle as a block-level parse. Otherwise not.
+				var parseAsInline = !(isStartOfInline && i === 0);
+				var textToParse = currentNode.content;
+				if (pluginOpts.renderWikiTextPragma !== "") {
+					textToParse = pluginOpts.renderWikiTextPragma + "\n" + textToParse;
 				}
+				var wikiParser = $tw.wiki.parseText("text/vnd.tiddlywiki", textToParse, {
+					parseAsInline: parseAsInline
+				});
+				var rs = wikiParser.tree;
+
+				// If we parsed as a block, but the root element the WikiText parser gave is a paragraph,
+				// we should discard the paragraph, since the way Remarkable nests its nodes, this "inline"
+				// node is always inside something else that's a block-level element
+				if (!parseAsInline
+					&& rs.length === 1
+					&& rs[0].type === "element"
+					&& rs[0].tag === "p"
+				) {
+					rs = rs[0].children;
+				}
+
+				// If the original text element started with a space, add it back in
+				if (rs.length > 0
+					&& rs[0].type === "text"
+					&& currentNode.content[0] === " "
+				) {
+					rs[0].text = " " + rs[0].text;
+				}
+				out = out.concat(rs);
+			}
+		} else {
+			console.error("Unknown node type: " + currentNode.type, currentNode);
+			out.push({
+				type: "text",
+				text: currentNode.content
+			});
+		}
+	}
+	return out;
 }
 
 var MarkdownParser = function(type, text, options) {
-	var dialect = options.wiki.getTiddlerText(CONFIG_DIALECT_TIDDLER,DEFAULT_DIALECT),
-		markdownTree = markdown.toHTMLTree(text,dialect),
-		node = $tw.utils.isArray(markdownTree[1]) ? markdownTree.slice(1) : markdownTree.slice(2);
-	this.tree = transformNodes(node);
+	var tree = md.parse(text, {});
+	//console.debug(tree);
+	tree = convertNodes(tree);
+	//console.debug(tree);
+
+	this.tree = tree;
 };
 
-/*
-
-[ 'html',
-  [ 'p', 'something' ],
-  [ 'h1',
-    'heading and ',
-    [ 'strong', 'other' ] ] ]
-
-*/
-
 exports["text/x-markdown"] = MarkdownParser;
 
 })();
-