Create plugin/parser structure, add creole parser

fs/html.go

@@ -1,19 +1,14 @@
 package fs
 
 import (
-	"bytes"
 	"fmt"
 	"io/ioutil"
 	"log"
 
+	"github.com/bouncepaw/mycorrhiza/plugin"
 	"github.com/bouncepaw/mycorrhiza/util"
-	"gopkg.in/russross/blackfriday.v2"
 )
 
-func markdownToHtml(md []byte) string {
-	return string(blackfriday.Run(NormalizeEOL(md)))
-}
-
 func (h *Hypha) asHtml() (string, error) {
 	rev := h.actual
 	ret := `<article class="page">
@@ -26,62 +21,17 @@ func (h *Hypha) asHtml() (string, error) {
 
 	contents, err := ioutil.ReadFile(rev.TextPath)
 	if err != nil {
-		log.Println("Failed to render", rev.FullName, ":", err)
+		log.Println("Failed to read contents of", rev.FullName, ":", err)
 		return "", err
 	}
 
 	// TODO: support more markups.
 	// TODO: support mycorrhiza extensions like transclusion.
-	switch rev.TextMime {
+	parser := plugin.ParserForMime(rev.TextMime)
-	case "text/markdown":
+	ret += parser(contents)
-		html := markdownToHtml(contents)
-		ret += string(html)
-	default:
-		ret += fmt.Sprintf(`<pre>%s</pre>`, contents)
-	}
 
 	ret += `
 </article>`
 
 	return ret, nil
 }
-
-// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF)
-// Code taken from here: https://github.com/go-gitea/gitea/blob/dc8036dcc680abab52b342d18181a5ee42f40318/modules/util/util.go#L68-L102
-// Gitea has MIT License
-//
-// We use it because md parser does not handle CRLF correctly. I don't know why, but CRLF appears sometimes.
-func NormalizeEOL(input []byte) []byte {
-	var right, left, pos int
-	if right = bytes.IndexByte(input, '\r'); right == -1 {
-		return input
-	}
-	length := len(input)
-	tmp := make([]byte, length)
-
-	// We know that left < length because otherwise right would be -1 from IndexByte.
-	copy(tmp[pos:pos+right], input[left:left+right])
-	pos += right
-	tmp[pos] = '\n'
-	left += right + 1
-	pos++
-
-	for left < length {
-		if input[left] == '\n' {
-			left++
-		}
-
-		right = bytes.IndexByte(input[left:], '\r')
-		if right == -1 {
-			copy(tmp[pos:], input[left:])
-			pos += length - left
-			break
-		}
-		copy(tmp[pos:pos+right], input[left:left+right])
-		pos += right
-		tmp[pos] = '\n'
-		left += right + 1
-		pos++
-	}
-	return tmp[:pos]
-}

go.mod

@@ -8,6 +8,7 @@ require (
 )
 
 require (
+	github.com/m4tty/cajun v0.0.0-20150303030909-35de273cc87b
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	gopkg.in/russross/blackfriday.v2 v2.0.1
 )

go.sum

@@ -4,6 +4,8 @@ github.com/gomarkdown/markdown v0.0.0-20200609195525-3f9352745725/go.mod h1:aii0
 github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
 github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/m4tty/cajun v0.0.0-20150303030909-35de273cc87b h1:aY3LtSBlkQahoWaPTytHcIFsDbeXFYMc4noRQ/N5Q+A=
+github.com/m4tty/cajun v0.0.0-20150303030909-35de273cc87b/go.mod h1:zFXkL7I5vIwKg4dxEA9025SLdIHu9qFX/cYTdUcusHc=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=

plugin/parser/creole.go

@@ -0,0 +1,11 @@
+package parser
+
+import (
+	"github.com/bouncepaw/mycorrhiza/util"
+	"github.com/m4tty/cajun"
+)
+
+func CreoleToHtml(creole []byte) string {
+	out, _ := cajun.Transform(string(util.NormalizeEOL(creole)))
+	return out
+}

plugin/parser/gemini.go

@@ -0,0 +1,7 @@
+package parser
+
+import ()
+
+func GeminiToHtml(gemini []byte) string {
+	return ""
+}

plugin/parser/markdown.go

@@ -0,0 +1,10 @@
+package parser
+
+import (
+	"github.com/bouncepaw/mycorrhiza/util"
+	"gopkg.in/russross/blackfriday.v2"
+)
+
+func MarkdownToHtml(md []byte) string {
+	return string(blackfriday.Run(util.NormalizeEOL(md)))
+}

plugin/plugin.go

@@ -0,0 +1,20 @@
+package plugin
+
+import (
+	"fmt"
+	"github.com/bouncepaw/mycorrhiza/plugin/parser"
+)
+
+func ParserForMime(mime string) func([]byte) string {
+	parsers := map[string]func([]byte) string{
+		"text/markdown": parser.MarkdownToHtml,
+		"text/creole":   parser.CreoleToHtml,
+		"text/gemini":   parser.GeminiToHtml,
+	}
+	if parserFunc, ok := parsers[mime]; ok {
+		return parserFunc
+	}
+	return func(contents []byte) string {
+		return fmt.Sprintf(`<pre><code>%s</code></pre>`, contents)
+	}
+}

util/util.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"bytes"
 	"strings"
 	"unicode"
 )
@@ -47,3 +48,43 @@ func CanonicalToDisplay(name string) (res string) {
 	}
 	return addColonPerhaps(res)
 }
+
+// NormalizeEOL will convert Windows (CRLF) and Mac (CR) EOLs to UNIX (LF)
+// Code taken from here: https://github.com/go-gitea/gitea/blob/dc8036dcc680abab52b342d18181a5ee42f40318/modules/util/util.go#L68-L102
+// Gitea has MIT License
+//
+// We use it because md parser does not handle CRLF correctly. I don't know why, but CRLF appears sometimes.
+func NormalizeEOL(input []byte) []byte {
+	var right, left, pos int
+	if right = bytes.IndexByte(input, '\r'); right == -1 {
+		return input
+	}
+	length := len(input)
+	tmp := make([]byte, length)
+
+	// We know that left < length because otherwise right would be -1 from IndexByte.
+	copy(tmp[pos:pos+right], input[left:left+right])
+	pos += right
+	tmp[pos] = '\n'
+	left += right + 1
+	pos++
+
+	for left < length {
+		if input[left] == '\n' {
+			left++
+		}
+
+		right = bytes.IndexByte(input[left:], '\r')
+		if right == -1 {
+			copy(tmp[pos:], input[left:])
+			pos += length - left
+			break
+		}
+		copy(tmp[pos:pos+right], input[left:left+right])
+		pos += right
+		tmp[pos] = '\n'
+		left += right + 1
+		pos++
+	}
+	return tmp[:pos]
+}

util/util_test.go

@@ -7,12 +7,15 @@ import (
 func TestWikilink(t *testing.T) {
 	atHypha := ":example/test"
 	results := map[string]string{
-		"foo":      "/foo",
+		"foo":                   "/foo",
-		"::foo":    "/:example/foo",
+		"::foo":                 "/:example/foo",
-		":bar/foo": "/:bar/foo",
+		":bar/foo":              "/:bar/foo",
-		"/baz":     "/:example/test/baz",
+		"/baz":                  "/:example/test/baz",
-		"./baz":    "/:example/test/baz",
+		"./baz":                 "/:example/test/baz",
-		"../qux":   "/:example/qux",
+		"../qux":                "/:example/qux",
+		"http://example.org":    "http://example.org",
+		"gemini://example.org":  "gemini://example.org",
+		"mailto:me@example.org": "mailto:me@example.org",
 	}
 	for link, expect := range results {
 		if res := Wikilink(link, atHypha); expect != res {

wiki/main/doc/plugin/2.markdown

@@ -0,0 +1,30 @@
+# Plugin system RFC
+MycorrhizaWiki engine does not provide all the functionality a wiki may need and not need. Instead, it relies on the system of plugins.
+
+## Types of plugins
+- **Parser.** They add support for displaying different MIME-types.
+- **Utilities.** They add hyphae to the `:spec` mycelium. These hyphae provide administrative functionality.
+- **Macros.** Something like [moinmoin ones](http://moinmo.in/HelpOnMacros), I guess.
+
+## Default plugins
+Default MycorrhizaWiki distributive is shipped with several plugins installed.
+
+- **parser/markdown.** Support for `text/markdown`. This parser is powered by [russross/blackfriday](https://github.com/russross/blackfriday); this parser is ok, I guess.
+- **parser/creole.** Support for `text/creole`. *Note:* there is no standard Creole MIME type. This parser is powered by [m4tty/cajun](https://github.com/m4tty/cajun); this library is somewhat outdated. Perhaps we'll reimplement it.
+- **parser/gemini.** Support for `text/gemini`. *Not implemented yet.*
+- *what about shipping BBcode? lol*
+- **utility/rename.** Renaming of non-user hyphae. *Not implemented yet.*
+- *what else?*
+- **macro/toc.** Table of contents. *Not implemented yet.*
+- *what else?*
+
+## Plugin implementation
+All plugins are written in Go and are compiled together with MycorrhizaWiki. If a wiki's admin decides to add a plugin, they shall recompile the engine with the plugin.
+
+> Reminds of [something](http://suckless.org/), right?
+
+*But compiling the engine just to add a plugin is stupid!!* Not really. Also, it makes the architecture more simple and secure.
+
+*What if an admin doesn't know how to program?* Plugin installation is basically limited to putting some files into a folder, editing the config and running a shell command. No programming required to install a plugin.
+
+See `plugin` directory at the root of the repo to get inspired by the present parsers.

wiki/main/doc/plugin/meta.json

@@ -14,6 +14,19 @@
 			"binary_mime": "",
 			"text_name": "1.markdown",
 			"binary_name": ""
+		},
+		"2": {
+			"tags": [
+				""
+			],
+			"name": "Plugin",
+			"comment": "Update :Main/Doc/Plugin",
+			"author": "",
+			"time": 1593882606,
+			"text_mime": "text/markdown",
+			"binary_mime": "",
+			"text_name": "2.markdown",
+			"binary_name": ""
 		}
 	}
 }

wiki/main/doc/wikilink/2.markdown

@@ -0,0 +1,49 @@
+# Wikilink RFC
+
+All parsers for MycorrhizaWiki provide hyperlink support. Usually, they follow HTML convention.
+
+- `http://example.org/absolute-path`
+- `/rooted-path`
+- `same-folder-path`
+- `../parent-folder-path`
+
+This is not really convenient for wikis where most of links are either rooted or links to children!
+
+All parsers of MycorrhizaWiki are expected to support these types of links and convert them to rooted paths.
+
+- `http://example.org/absolute-path`
+- `hypha in main mycelium`
+- `::hypha in the same mycelium`
+- `:mycelium/hypha in an explicit mycelium`
+- `/subhypha`
+- `./subhypha`
+- `../sibling-hypha`
+
+**TODO:** create a package that implements this thing. NB: to generate a correct link, it is required to know full name of hypha where the link is used.
+
+## Markdown extension
+
+> This is an extension to markdown's syntax that is used in MycorrhizaWiki and nowhere else.
+
+Text wrapped in `[[` and `]]` is a link that has same text and url. *For some reason it's not possible in Markdown without duplicating url*
+
+```
+[[x]] == [x](x)
+```
+
+## Examples
+
+All examples assume that `:example/test` is the current hypha.
+
+```
+wikilink              actual path
+foo                   == /foo
+::foo                 == /:example/foo
+:bar/foo              == /:bar/foo
+/baz                  == /:example/test/baz
+./baz                 == /:example/test/baz
+../qux                == /:example/qux
+http://example.org    == http://example.org
+gemini://example.org  == gemini://example.org
+mailto:me@example.org == mailto:me@example.org
+```

wiki/main/doc/wikilink/meta.json

@@ -14,6 +14,19 @@
 			"binary_mime": "",
 			"text_name": "1.markdown",
 			"binary_name": ""
+		},
+		"2": {
+			"tags": [
+				""
+			],
+			"name": "Wikilink",
+			"comment": "Update :Main/Doc/Wikilink",
+			"author": "",
+			"time": 1593875778,
+			"text_mime": "text/markdown",
+			"binary_mime": "",
+			"text_name": "2.markdown",
+			"binary_name": ""
 		}
 	}
 }

wiki/main/testcreole/1.txt

@@ -0,0 +1,135 @@
+//This test was taken from http://www.wikicreole.org/wiki/JSPWikiTestCases?skin=raw//
+
+The following is a test of WikiCreole:
+
+= Top-level heading (1)
+== This a test for creole 0.1 (2)
+=== This is a Subheading (3)
+==== Subsub (4)
+===== Subsubsub (5)
+
+The ending equal signs should not be displayed:
+
+= Top-level heading (1) =
+== This a test for creole 0.1 (2) ==
+=== This is a Subheading (3) ===
+==== Subsub (4) ====
+===== Subsubsub (5) =====
+
+
+You can make things **bold** or //italic// or **//both//** or //**both**//.
+
+Character formatting extends across line breaks: **bold,
+this is still bold. This line deliberately does not end in star-star.
+
+Not bold. Character formatting does not cross paragraph boundaries.
+
+You can use [[internal links]] or [[http://www.wikicreole.org|external links]],
+give the link a [[internal links|different]] name.
+
+Here's another sentence: This wisdom is taken from [[Ward Cunningham's]]
+[[http://www.c2.com/doc/wikisym/WikiSym2006.pdf|Presentation at the Wikisym 06]].
+
+Here's a external link without a description: [[http://www.wikicreole.org]]
+
+Free links without braces should be rendered as well, like http://www.wikicreole.org/ and http://www.wikicreole.org/users/~example. 
+
+Creole1.0 specifies that http://bar and ftp://bar should not render italic,
+something like foo://bar should render as italic.
+
+You can use this to draw a line to separate the page:
+----
+
+You can use lists, start it at the first column for now, please...
+
+unnumbered lists are like
+* item a
+* item b
+* **bold item c**
+
+blank space is also permitted before lists like:
+  *   item a
+ * item b
+* item c
+ ** item c.a
+
+or you can number them
+# [[item 1]]
+# item 2
+# // italic item 3 //
+    ## item 3.1
+  ## item 3.2
+
+up to five levels
+* 1
+** 2
+*** 3
+**** 4
+***** 5
+
+* You can have
+multiline list items
+* this is a second multiline
+list item
+
+You can use nowiki syntax if you would like do stuff like this:
+
+{{{
+Guitar Chord C:
+
+||---|---|---|
+||-1-|---|---|
+||---|---|---|
+||---|-2-|---|
+||---|---|-3-|
+||---|---|---|
+~}}}
+
+Note: if you look at the source code of the above, you see the escape char (tilde, ~ )
+being used to escape the closing triple curly braces. This is to do nesting because
+all this text is enclosed in nowiki markup.
+}}}
+
+You can also use it inline nowiki {{{ in a sentence }}} like this.
+
+= Escapes =
+Normal Link: http://wikicreole.org/ - now same link, but escaped: ~http://wikicreole.org/ 
+
+Normal asterisks: ~**not bold~**
+
+a tilde alone: ~
+
+a tilde escapes itself: ~~xxx
+
+=== Creole 0.2 ===
+
+This should be a flower with the ALT text "this is a flower" if your wiki supports ALT text on images:
+
+{{Red_Flower.jpg|here is a red flower}}
+
+=== Creole 0.4 ===
+
+Tables are done like this:
+
+|=header col1|=header col2| 
+|col1|col2| 
+|you         |can                                       | 
+|also        |align\\ it.                               | 
+|links       |[[http://www.wikicreole.org/|Should Work]]|
+
+You can format an address by simply forcing linebreaks:
+
+My contact dates:\\
+Pone: xyz\\
+Fax: +45\\
+Mobile: abc
+
+=== Creole 0.5 ===
+
+|= Header title               |= Another header title     |
+| {{{ //not italic text// }}} | {{{ **not bold text** }}} |
+| //italic text//             | **  bold text **          |
+
+=== Creole 1.0 ===
+
+If interwiki links are setup in your wiki, this links to the WikiCreole page about Creole 1.0 test cases: [[WikiCreole:Creole1.0TestCases]].

wiki/main/testcreole/2.txt

@@ -0,0 +1,135 @@
+//This test was taken from http://www.wikicreole.org/wiki/JSPWikiTestCases?skin=raw//
+
+The following is a test of WikiCreole:
+
+= Top-level heading (1)
+== This a test for creole 0.1 (2)
+=== This is a Subheading (3)
+==== Subsub (4)
+===== Subsubsub (5)
+
+The ending equal signs should not be displayed:
+
+= Top-level heading (1) =
+== This a test for creole 0.1 (2) ==
+=== This is a Subheading (3) ===
+==== Subsub (4) ====
+===== Subsubsub (5) =====
+
+
+You can make things **bold** or //italic// or **//both//** or //**both**//.
+
+Character formatting extends across line breaks: **bold,
+this is still bold. This line deliberately does not end in star-star.
+
+Not bold. Character formatting does not cross paragraph boundaries.
+
+You can use [[internal links]] or [[http://www.wikicreole.org|external links]],
+give the link a [[internal links|different]] name.
+
+Here's another sentence: This wisdom is taken from [[Ward Cunningham's]]
+[[http://www.c2.com/doc/wikisym/WikiSym2006.pdf|Presentation at the Wikisym 06]].
+
+Here's a external link without a description: [[http://www.wikicreole.org]]
+
+Free links without braces should be rendered as well, like http://www.wikicreole.org/ and http://www.wikicreole.org/users/~example. 
+
+Creole1.0 specifies that http://bar and ftp://bar should not render italic,
+something like foo://bar should render as italic.
+
+You can use this to draw a line to separate the page:
+----
+
+You can use lists, start it at the first column for now, please...
+
+unnumbered lists are like
+* item a
+* item b
+* **bold item c**
+
+blank space is also permitted before lists like:
+  *   item a
+ * item b
+* item c
+ ** item c.a
+
+or you can number them
+# [[item 1]]
+# item 2
+# // italic item 3 //
+    ## item 3.1
+  ## item 3.2
+
+up to five levels
+* 1
+** 2
+*** 3
+**** 4
+***** 5
+
+* You can have
+multiline list items
+* this is a second multiline
+list item
+
+You can use nowiki syntax if you would like do stuff like this:
+
+{{{
+Guitar Chord C:
+
+||---|---|---|
+||-1-|---|---|
+||---|---|---|
+||---|-2-|---|
+||---|---|-3-|
+||---|---|---|
+~}}}
+
+Note: if you look at the source code of the above, you see the escape char (tilde, ~ )
+being used to escape the closing triple curly braces. This is to do nesting because
+all this text is enclosed in nowiki markup.
+}}}
+
+You can also use it inline nowiki {{{ in a sentence }}} like this.
+
+= Escapes =
+Normal Link: http://wikicreole.org/ - now same link, but escaped: ~http://wikicreole.org/ 
+
+Normal asterisks: ~**not bold~**
+
+a tilde alone: ~
+
+a tilde escapes itself: ~~xxx
+
+=== Creole 0.2 ===
+
+This should be a flower with the ALT text "this is a flower" if your wiki supports ALT text on images:
+
+{{/Red_Flower.jpg|here is a red flower}}
+
+=== Creole 0.4 ===
+
+Tables are done like this:
+
+|=header col1|=header col2| 
+|col1|col2| 
+|you         |can                                       | 
+|also        |align\\ it.                               | 
+|links       |[[http://www.wikicreole.org/|Should Work]]|
+
+You can format an address by simply forcing linebreaks:
+
+My contact dates:\\
+Pone: xyz\\
+Fax: +45\\
+Mobile: abc
+
+=== Creole 0.5 ===
+
+|= Header title               |= Another header title     |
+| {{{ //not italic text// }}} | {{{ **not bold text** }}} |
+| //italic text//             | **  bold text **          |
+
+=== Creole 1.0 ===
+
+If interwiki links are setup in your wiki, this links to the WikiCreole page about Creole 1.0 test cases: [[WikiCreole:Creole1.0TestCases]].

wiki/main/testcreole/meta.json

@@ -0,0 +1,32 @@
+{
+	"views": 0,
+	"deleted": false,
+	"revisions": {
+		"1": {
+			"tags": [
+				""
+			],
+			"name": "Testcreole",
+			"comment": "Update :Main/Testcreole",
+			"author": "",
+			"time": 1593881943,
+			"text_mime": "text/creole",
+			"binary_mime": "",
+			"text_name": "1.txt",
+			"binary_name": ""
+		},
+		"2": {
+			"tags": [
+				""
+			],
+			"name": "Testcreole",
+			"comment": "Update :Main/Testcreole",
+			"author": "",
+			"time": 1593882073,
+			"text_mime": "text/creole",
+			"binary_mime": "",
+			"text_name": "2.txt",
+			"binary_name": ""
+		}
+	}
+}