first commit

This commit is contained in:
osmarks 2021-01-06 16:09:48 +00:00
commit 3ad3b1aad0
13 changed files with 518 additions and 0 deletions

2
.env Normal file
View File

@ -0,0 +1,2 @@
secretKey=so many apioforms
staticDir=static

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
minoteaur.sqlite3
dump.sql
minoteaur

7
LICENSE Normal file
View File

@ -0,0 +1,7 @@
Copyright 2020 osmarks
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

4
README.md Normal file
View File

@ -0,0 +1,4 @@
# minoteaur
A wiki-style personal note-taking application. Written in Nim with Prologue, Karax, SQLite.
Still highly experimental, and contains mildly horrific code. Please do not use this for anything serious right now. I may arbitrarily break backward compatibility and change around the database.

22
minoteaur.nimble Normal file
View File

@ -0,0 +1,22 @@
# Package
version = "0.1.0"
author = "osmarks"
description = "A notes thing™."
license = "MIT"
srcDir = "src"
bin = @["minoteaur"]
# Dependencies
requires "nim >= 1.4.2"
requires "prologue >= 0.4"
requires "karax >= 1.2.1"
requires "https://github.com/GULPF/tiny_sqlite#2944bc7"
requires "zstd >= 0.5.0"
requires "https://github.com/osmarks/nim-cmark-gfm"
#requires "cmark >= 0.1.0"
requires "regex >= 0.18.0"
# seemingly much faster than standard library Levenshtein distance module
requires "nimlevenshtein >= 0.1.0"
#requires "gara >= 0.2.0"

1
nim.cfg Normal file
View File

@ -0,0 +1 @@
--threads:on

137
src/domain.nim Normal file
View File

@ -0,0 +1,137 @@
import tiny_sqlite
import logging
import options
import times
import zstd/compress
import zstd/decompress
import sequtils
import strutils except splitWhitespace
import json
import std/jsonutils
import nimlevenshtein
import sugar
import unicode
func timeToTimestamp*(t: Time): int64 = toUnix(t) * 1000 + (nanosecond(t) div 1000000)
func timestampToTime*(ts: int64): Time = initTime(ts div 1000, (ts mod 1000) * 1000000)
func timestampToStr*(t: Time): string = intToStr(int(timeToTimestamp(t)))
# store time as milliseconds
proc toDbValue(t: Time): DbValue = DbValue(kind: sqliteInteger, intVal: timeToTimestamp(t))
proc fromDbValue(value: DbValue, T: typedesc[Time]): Time = timestampToTime(value.intVal)
let migrations = @[
"""CREATE TABLE pages (
page TEXT NOT NULL PRIMARY KEY,
updated INTEGER NOT NULL,
created INTEGER NOT NULL
);
CREATE TABLE revisions (
page TEXT NOT NULL REFERENCES pages(page),
timestamp INTEGER NOT NULL,
meta TEXT NOT NULL,
fullData BLOB
);"""
]
type
Encoding = enum
encPlain = 0, encZstd = 1
RevisionType = enum
rtNewContent = 0
RevisionMeta = object
case typ*: RevisionType
of rtNewContent:
encoding*: Encoding
editDistance*: Option[int]
size*: Option[int]
words*: Option[int]
Revision = object
meta*: Revisionmeta
time*: Time
var logger = newConsoleLogger()
proc migrate*(db: DbConn) =
let currentVersion = fromDbValue(get db.value("PRAGMA user_version"), int)
for mid in (currentVersion + 1) .. migrations.len:
db.transaction:
logger.log(lvlInfo, "Migrating to schema " & $mid)
db.execScript migrations[mid - 1]
# for some reason this pragma does not work using normal parameter binding
db.exec("PRAGMA user_version = " & $mid)
logger.log(lvlDebug, "DB ready")
type
Page = object
page*, content*: string
created*, updated*: Time
proc parse*(s: string, T: typedesc): T = fromJson(result, parseJSON(s), Joptions(allowExtraKeys: true, allowMissingKeys: true))
proc processFullRevisionRow(row: ResultRow): (RevisionMeta, string) =
let (metaJSON, full) = row.unpack((string, seq[byte]))
let meta = parse(metaJSON, RevisionMeta)
var content = cast[string](full)
if meta.encoding == encZstd:
content = cast[string](decompress(content))
(meta, content)
proc fetchPage*(db: DbConn, page: string, revision: Option[Time] = none(Time)): Option[Page] =
# retrieve row for page
db.one("SELECT updated, created FROM pages WHERE page = ?", page).flatMap(proc(row: ResultRow): Option[Page] =
let (updated, created) = row.unpack((Time, Time))
let rev =
if revision.isSome: db.one("SELECT meta, fullData FROM revisions WHERE page = ? AND json_extract(meta, '$.typ') = 0 AND timestamp = ?", page, revision)
else: db.one("SELECT meta, fullData FROM revisions WHERE page = ? AND json_extract(meta, '$.typ') = 0 ORDER BY timestamp DESC LIMIT 1", page)
rev.map(proc(row: ResultRow): Page =
let (meta, content) = processFullRevisionRow(row)
Page(page: page, created: created, updated: updated, content: content)
)
)
# count words, defined as things separated by whitespace which are not purely Markdown-ish punctuation characters
# alternative definitions may include dropping number-only words, and/or splitting at full stops too
func wordCount(s: string): int =
for word in splitWhitespace(s):
if len(word) == 0: continue
for bytechar in word:
if not (bytechar in {'#', '*', '-', '>', '`', '|', '-'}):
inc result
break
proc updatePage*(db: DbConn, page: string, content: string) =
let previous = fetchPage(db, page).map(p => p.content).get("")
let compressed = compress(content, level=10)
var enc = encPlain
var data = cast[seq[byte]](content)
if len(compressed) < len(data):
enc = encZstd
data = compressed
let meta = $toJson(RevisionMeta(typ: rtNewContent, encoding: enc,
editDistance: some distance(previous, content), size: some len(content), words: some wordCount(content)))
let ts = getTime()
db.transaction:
db.exec("INSERT INTO revisions VALUES (?, ?, ?, ?)", page, ts, meta, data)
db.exec("INSERT INTO pages VALUES (?, ?, ?) ON CONFLICT (page) DO UPDATE SET updated = ?", page, ts, ts, ts)
proc fetchRevisions*(db: DbConn, page: string): seq[Revision] =
db.all("SELECT timestamp, meta FROM revisions WHERE page = ? ORDER BY timestamp DESC", page).map(proc (row: ResultRow): Revision =
let (ts, metaJSON) = row.unpack((Time, string))
Revision(time: ts, meta: parse(metaJSON, RevisionMeta))
)
proc processRevisionRow(r: ResultRow): Revision =
let (ts, meta) = r.unpack((Time, string))
Revision(time: ts, meta: parse(meta, RevisionMeta))
proc adjacentRevisions*(db: DbConn, page: string, ts: Time): (Option[Revision], Option[Revision]) =
# revision after given timestamp
let next = db.one("SELECT timestamp, meta FROM revisions WHERE page = ? AND json_extract(meta, '$.typ') = 0 AND timestamp > ? ORDER BY timestamp ASC LIMIT 1", page, ts)
# revision before given timestamp
let prev = db.one("SELECT timestamp, meta FROM revisions WHERE page = ? AND json_extract(meta, '$.typ') = 0 AND timestamp < ? ORDER BY timestamp DESC LIMIT 1", page, ts)
(next.map(processRevisionRow), prev.map(processRevisionRow))

72
src/md.nim Normal file
View File

@ -0,0 +1,72 @@
import karax/[karaxdsl, vdom]
import cmark/native
# the builtin re library would probably be better for this - it can directly take cstrings (so better perf when dealing with the cstrings from cmark) and may be faster
# unfortunately it does not expose a findAll thing which returns the *positions* of everything for some weird reason
import regex
cmark_gfm_core_extensions_ensure_registered()
func wikilink(page, linkText: string): string =
let vdom = buildHtml(a(href=page, class="wikilink")): text linkText
$vdom
proc pushNodeAfter(ty: NodeType, content: string, pushAfter: NodePtr) =
let node = cmark_node_new(ty)
assert cmark_node_set_literal(node, content) == 1
assert cmark_node_insert_before(pushAfter, node) == 1
proc renderToHtml*(input: string): string =
let wlRegex = re"\[\[([^:\]]+):?([^\]]+)?\]\]"
let opt = CMARK_OPT_UNSAFE or CMARK_OPT_FOOTNOTES or CMARK_OPT_STRIKETHROUGH_DOUBLE_TILDE or CMARK_OPT_TABLE_PREFER_STYLE_ATTRIBUTES
let
str: cstring = input
len: csize_t = len(input).csize_t
parser: ParserPtr = cmark_parser_new(opt.cint)
if parser == nil: raise newException(CatchableError, "failed to initialize parser")
defer: cmark_parser_free(parser)
for ext in @["table", "strikethrough"]:
let e: cstring = ext
let eptr = cmark_find_syntax_extension(e)
if eptr == nil: raise newException(LibraryError, "failed to find extension " & ext)
if cmark_parser_attach_syntax_extension(parser, eptr) == 0: raise newException(CatchableError, "failed to attach extension " & ext)
cmark_parser_feed(parser, str, len)
let doc = cmark_parser_finish(parser)
defer: cmark_node_free(doc)
if doc == nil: raise newException(CatchableError, "parsing failed")
block:
let iter = cmark_iter_new(doc)
defer: cmark_iter_free(iter)
while true:
let evType = cmark_iter_next(iter)
if evType == etDone: break
let node: NodePtr = cmark_iter_get_node(iter)
if cmark_node_get_type(node) == ntText:
let ntext = $cmark_node_get_literal(node)
# check for wikilinks in text node
let matches = findAll(ntext, wlRegex)
# if there are any, put in the appropriate HTML nodes
if len(matches) > 0:
var lastix = 0
for match in matches:
let page = ntext[match.captures[0][0]] # I don't know why this doesn't use Option. Perhaps sometimes there are somehow > 1 ranges.
# if there is a separate linkText field, use this, otherwise just use the page
let linkText =
if len(match.captures[1]) > 0: ntext[match.captures[1][0]]
else: page
let html = wikilink(page, linkText)
# push text before this onto the tree, as well as the HTML of the wikilink
pushNodeAfter(ntText, ntext[lastix..<match.boundaries.a], node)
pushNodeAfter(ntHtmlInline, html, node)
lastix = match.boundaries.b + 1
# push final text, if relevant
if lastix != len(ntext) - 1: pushNodeAfter(ntText, ntext[lastix..<len(ntext)], node)
cmark_node_free(node)
let html: cstring = cmark_render_html(doc, opt.cint, cmark_parser_get_syntax_extensions(parser))
defer: free(html)
result = $html

153
src/minoteaur.nim Normal file
View File

@ -0,0 +1,153 @@
import prologue
import prologue/middlewares/staticfile
import karax/[karaxdsl, vdom]
import prologue/middlewares/sessions/signedcookiesession
from uri import decodeUrl, encodeUrl
import tiny_sqlite
import options
import times
import sugar
import strutils
from ./domain import nil
from ./md import nil
let
env = loadPrologueEnv(".env")
settings = newSettings(
appName = "minoteaur",
debug = env.getOrDefault("debug", true),
port = Port(env.getOrDefault("port", 7600)),
secretKey = env.getOrDefault("secretKey", "")
)
func navButton(content: string, href: string, class: string): VNode = buildHtml(a(class="link-button " & class, href=href)): text content
func base(title: string, navItems: seq[VNode], bodyItems: VNode): string =
let vnode = buildHtml(html):
head:
link(rel="stylesheet", href="/static/style.css")
meta(charset="utf8")
meta(name="viewport", content="width=device-width,initial-scale=1.0")
title: text title
body:
main:
nav:
for n in navItems: n
tdiv(class="header"):
h1: text title
bodyItems
$vnode
domain.migrate(openDatabase("./minoteaur.sqlite3"))
type
AppContext = ref object of Context
db: DbConn
# store thread's DB connection
var db {.threadvar.}: Option[DbConn]
proc dbMiddleware(): HandlerAsync =
result = proc(ctx: AppContext) {.async.} =
# open new DB connection for thread if there isn't one
if db.isNone:
db = some openDatabase("./minoteaur.sqlite3")
# close DB connection on thread exit
onThreadDestruction(proc() =
try: db.get().close()
except: discard)
ctx.db = get db
await switch(ctx)
proc displayTime(t: Time): string = t.format("uuuu-MM-dd HH:mm:ss", utc())
func pageUrlFor(ctx: AppContext, route: string, page: string, query: openArray[(string, string)] = @[]): string = ctx.urlFor(route, { "page": encodeUrl(page) }, query)
func pageButton(ctx: AppContext, route: string, page: string, label: string, query: openArray[(string, string)] = @[]): VNode = navButton(label, pageUrlFor(ctx, route, page, query), route)
proc edit(ctx: AppContext) {.async.} =
let page = decodeUrl(ctx.getPathParams("page"))
let pageData = domain.fetchPage(ctx.db, page)
let html =
buildHtml(form(`method`="post", class="edit-form")):
textarea(name="content"): text pageData.map(p => p.content).get("")
input(`type`="submit", value="Save", name="action", class="save")
let verb = if pageData.isSome: "Editing " else: "Creating "
resp base(verb & page, @[pageButton(ctx, "view-page", page, "View"), pageButton(ctx, "page-revisions", page, "Revisions")], html)
proc revisions(ctx: AppContext) {.async.} =
let page = decodeUrl(ctx.getPathParams("page"))
let revs = domain.fetchRevisions(ctx.db, page)
let html =
buildHtml(table(class="rev-table")):
tr:
th: text "Time"
th: text "Changes"
th: text "Size"
th: text "Words"
for rev in revs:
tr:
td(class="ts"):
a(href=ctx.urlFor("view-page", { "page": encodeUrl(page) }, { "ts": domain.timestampToStr(rev.time) })):
text displayTime(rev.time)
td: text rev.meta.editDistance.map(x => $x).get("")
td: text rev.meta.size.map(x => formatSize(x)).get("")
td: text rev.meta.words.map(x => $x).get("")
resp base("Revisions of " & page, @[pageButton(ctx, "view-page", page, "View"), pageButton(ctx, "edit-page", page, "Edit")], html)
proc handleEdit(ctx: AppContext) {.async.} =
let page = decodeUrl(ctx.getPathParams("page"))
domain.updatePage(ctx.db, page, ctx.getFormParams("content"))
resp redirect(pageUrlFor(ctx, "view-page", page))
proc view(ctx: AppContext) {.async.} =
let page = decodeUrl(ctx.getPathParams("page"))
let rawRevision = ctx.getQueryParams("ts")
let viewSource = ctx.getQueryParams("source") != ""
let revisionTs = if rawRevision == "": none(Time) else: some domain.timestampToTime(parseInt rawRevision)
let viewingOldRevision = revisionTs.isSome
let pageData = domain.fetchPage(ctx.db, page, revisionTs)
if pageData.isNone:
resp redirect(pageUrlFor(ctx, "edit-page", page))
else:
let pageData = get pageData
let mainBody = if viewSource: buildHtml(pre): text pageData.content else: verbatim md.renderToHtml(pageData.content)
if revisionTs.isNone:
let html =
buildHtml(tdiv):
tdiv(class="timestamp"):
text "Updated "
text displayTime(pageData.updated)
tdiv(class="timestamp"):
text "Created "
text displayTime(pageData.created)
tdiv(class="md"): mainBody
resp base(page, @[pageButton(ctx, "edit-page", page, "Edit"), pageButton(ctx, "page-revisions", page, "Revisions")], html)
else:
let rts = get revisionTs
let (next, prev) = domain.adjacentRevisions(ctx.db, page, rts)
let html =
buildHtml(tdiv):
tdiv(class="timestamp"):
text "As of "
text displayTime(rts)
tdiv(class="md"): mainBody
var buttons = @[pageButton(ctx, "edit-page", page, "Edit"), pageButton(ctx, "page-revisions", page, "Revisions"), pageButton(ctx, "view-page", page, "Latest")]
if next.isSome: buttons.add(pageButton(ctx, "next-page", page, "Next", { "ts": domain.timestampToStr (get next).time }))
if prev.isSome: buttons.add(pageButton(ctx, "prev-page", page, "Previous", { "ts": domain.timestampToStr (get prev).time }))
resp base(page, buttons, html)
proc favicon(ctx: Context) {.async.} = resp "bee"
proc index(ctx: Context) {.async.} = resp "bee(s)"
var app = newApp(settings = settings)
app.use(@[staticFileMiddleware("static"), sessionMiddleware(settings), extendContextMiddleware(AppContext), dbMiddleware()])
app.get("/", index)
app.get("/favicon.ico", favicon)
app.get("/{page}/edit", edit, name="edit-page")
app.get("/{page}/revisions", revisions, name="page-revisions")
app.post("/{page}/edit", handleEdit, name="handle-edit")
app.get("/{page}/", view, name="view-page")
app.run()

113
src/style.sass Executable file
View File

@ -0,0 +1,113 @@
html
scrollbar-color: black lightgray
*
box-sizing: border-box
body
font-family: "Fira Sans", "Noto Sans", "Segoe UI", Verdana, sans-serif
font-weight: 300
margin: 0
min-height: 100vh
main
width: 50em
padding: 0 1em 1em 1em
margin-left: auto
margin-right: auto
strong
font-weight: 600
h1, h2, h3, h4, h5, h6
&:first-of-type
border-bottom: 1px solid gray
margin: 0
font-weight: 500
a
color: inherit
:not(pre) > code
background: black
color: white
padding: 0.3em
ul, ol
padding-left: 1em
ul
list-style-type: square
blockquote
border-left: 0.3em solid black
padding-left: 0.3em
table
border-collapse: collapse
th
background: black
color: white
font-weight: normal
td, th
padding: 0.2em 0.5em
td
border: 1px solid gray
.rev-table
width: 100%
td
border: none
&.ts
white-space: nowrap
.header
margin-bottom: 0.5em
.md
margin-top: 0.5em
> *, p
margin: 0 0 0.5em 0
nav
margin-bottom: 0.5em
.timestamp
color: gray
a.wikilink
text-decoration: none
color: #01a049
font-style: italic
&:hover
text-decoration: underline
.link-button, button, input[type=submit]
border: none
padding: 0.75em
background: gray
text-align: center
text-decoration: none
color: black
display: inline-block
font-size: 1rem
&:hover
// very hacky way to make the button darker when hovered
background-image: linear-gradient(rgba(0,0,0,.2), rgba(0,0,0,.2))
&.view-page
background-color: #76cd26
&.edit-page
background-color: #75bbfd
&.page-revisions
background-color: #f97306
&.save
background-color: #06c2ac
&.next-page
background-color: #5170d7
&.prev-page
background-color: #bc13fe
.edit-form
textarea
resize: vertical
width: 100%
height: 70vh
border: 1px solid gray

1
static/style.css Normal file
View File

@ -0,0 +1 @@
html{scrollbar-color:#000 #d3d3d3}*{box-sizing:border-box}body{font-family:"Fira Sans","Noto Sans","Segoe UI",Verdana,sans-serif;font-weight:300;margin:0;min-height:100vh}main{width:50em;padding:0 1em 1em 1em;margin-left:auto;margin-right:auto}strong{font-weight:600}h1,h2,h3,h4,h5,h6{margin:0;font-weight:500}h1:first-of-type,h2:first-of-type,h3:first-of-type,h4:first-of-type,h5:first-of-type,h6:first-of-type{border-bottom:1px solid gray}h1 a,h2 a,h3 a,h4 a,h5 a,h6 a{color:inherit}:not(pre)>code{background:#000;color:#fff;padding:.3em}ul,ol{padding-left:1em}ul{list-style-type:square}blockquote{border-left:.3em solid #000;padding-left:.3em}table{border-collapse:collapse}table th{background:#000;color:#fff;font-weight:normal}table td,table th{padding:.2em .5em}table td{border:1px solid gray}.rev-table{width:100%}.rev-table td{border:none}.rev-table td.ts{white-space:nowrap}.header{margin-bottom:.5em}.md{margin-top:.5em}.md>*,.md p{margin:0 0 .5em 0}nav{margin-bottom:.5em}.timestamp{color:gray}a.wikilink{text-decoration:none;color:#01a049;font-style:italic}a.wikilink:hover{text-decoration:underline}.link-button,button,input[type=submit]{border:none;padding:.75em;background:gray;text-align:center;text-decoration:none;color:#000;display:inline-block;font-size:1rem}.link-button:hover,button:hover,input[type=submit]:hover{background-image:linear-gradient(rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0.2))}.link-button.view-page,button.view-page,input[type=submit].view-page{background-color:#76cd26}.link-button.edit-page,button.edit-page,input[type=submit].edit-page{background-color:#75bbfd}.link-button.page-revisions,button.page-revisions,input[type=submit].page-revisions{background-color:#f97306}.link-button.save,button.save,input[type=submit].save{background-color:#06c2ac}.link-button.next-page,button.next-page,input[type=submit].next-page{background-color:#5170d7}.link-button.prev-page,button.prev-page,input[type=submit].prev-page{background-color:#bc13fe}.edit-form textarea{resize:vertical;width:100%;height:70vh;border:1px solid gray}/*# sourceMappingURL=style.css.map */

1
static/style.css.map Normal file
View File

@ -0,0 +1 @@
{"version":3,"sourceRoot":"","sources":["../src/style.sass"],"names":[],"mappings":"AAAA,KACI,6BAEJ,EACI,sBAEJ,KACI,kEACA,gBACA,SACA,iBAEJ,KACI,WACA,sBACA,iBACA,kBAEJ,OACI,gBAEJ,kBAGI,SACA,gBAHA,sGACI,6BAGJ,8BACI,cAER,eACI,gBACA,WACA,aAEJ,MACI,iBACJ,GACI,uBACJ,WACI,4BACA,kBACJ,MACI,yBAEA,SACI,gBACA,WACA,mBACJ,kBACI,kBACJ,SACI,sBAER,WACI,WACA,cACI,YACA,iBACI,mBAEZ,QACI,mBAEJ,IACI,gBACA,YACI,kBAER,IACI,mBAEJ,WACI,WAEJ,WACI,qBACA,cACA,kBACA,iBACI,0BAER,uCACI,YACA,cACA,gBACA,kBACA,qBACA,WACA,qBACA,eACA,yDAEI,yEAEJ,qEACI,yBACJ,qEACI,yBACJ,oFACI,yBACJ,sDACI,yBACJ,qEACI,yBACJ,qEACI,yBAGJ,oBACI,gBACA,WACA,YACA","file":"style.css"}

2
watch-css.sh Executable file
View File

@ -0,0 +1,2 @@
#!/bin/sh
npx sass --watch -s compressed src/style.sass:static/style.css