New blog post and redesigned iconography
Before Width: | Height: | Size: 6.9 KiB |
Before Width: | Height: | Size: 349 KiB After Width: | Height: | Size: 1.6 MiB |
Before Width: | Height: | Size: 22 KiB |
Before Width: | Height: | Size: 589 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 18 KiB |
Before Width: | Height: | Size: 554 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 13 KiB |
Before Width: | Height: | Size: 348 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 11 KiB |
Before Width: | Height: | Size: 471 KiB After Width: | Height: | Size: 1.7 MiB |
Before Width: | Height: | Size: 5.4 KiB |
Before Width: | Height: | Size: 335 KiB After Width: | Height: | Size: 1.3 MiB |
Before Width: | Height: | Size: 22 KiB |
Before Width: | Height: | Size: 524 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 12 KiB |
Before Width: | Height: | Size: 490 KiB After Width: | Height: | Size: 2.0 MiB |
BIN
assets/images/grafana1.png
Normal file
After Width: | Height: | Size: 204 KiB |
BIN
assets/images/grafana2.png
Normal file
After Width: | Height: | Size: 97 KiB |
BIN
assets/images/grafana3.png
Normal file
After Width: | Height: | Size: 277 KiB |
Before Width: | Height: | Size: 8.8 KiB |
Before Width: | Height: | Size: 413 KiB After Width: | Height: | Size: 1.4 MiB |
Before Width: | Height: | Size: 4.8 KiB |
Before Width: | Height: | Size: 378 KiB After Width: | Height: | Size: 1.1 MiB |
Before Width: | Height: | Size: 11 KiB |
Before Width: | Height: | Size: 431 KiB After Width: | Height: | Size: 1.5 MiB |
Before Width: | Height: | Size: 8.9 KiB |
Before Width: | Height: | Size: 600 KiB After Width: | Height: | Size: 1.8 MiB |
Before Width: | Height: | Size: 5.6 KiB |
Before Width: | Height: | Size: 362 KiB After Width: | Height: | Size: 1.5 MiB |
Before Width: | Height: | Size: 8.8 KiB |
Before Width: | Height: | Size: 458 KiB After Width: | Height: | Size: 1.4 MiB |
Before Width: | Height: | Size: 14 KiB |
Before Width: | Height: | Size: 616 KiB After Width: | Height: | Size: 2.1 MiB |
Before Width: | Height: | Size: 7.4 KiB |
Before Width: | Height: | Size: 355 KiB After Width: | Height: | Size: 1.6 MiB |
Before Width: | Height: | Size: 11 KiB |
Before Width: | Height: | Size: 429 KiB After Width: | Height: | Size: 1.3 MiB |
Before Width: | Height: | Size: 10 KiB |
Before Width: | Height: | Size: 339 KiB After Width: | Height: | Size: 1.6 MiB |
Before Width: | Height: | Size: 14 KiB |
Before Width: | Height: | Size: 442 KiB After Width: | Height: | Size: 1.7 MiB |
Before Width: | Height: | Size: 5.5 KiB |
Before Width: | Height: | Size: 182 KiB After Width: | Height: | Size: 1.5 MiB |
Before Width: | Height: | Size: 6.3 KiB |
Before Width: | Height: | Size: 440 KiB After Width: | Height: | Size: 1.6 MiB |
Before Width: | Height: | Size: 14 KiB |
Before Width: | Height: | Size: 517 KiB After Width: | Height: | Size: 1.5 MiB |
Before Width: | Height: | Size: 18 KiB |
Before Width: | Height: | Size: 558 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 16 KiB |
Before Width: | Height: | Size: 559 KiB After Width: | Height: | Size: 1.8 MiB |
Before Width: | Height: | Size: 6.3 KiB |
Before Width: | Height: | Size: 493 KiB After Width: | Height: | Size: 932 KiB |
Before Width: | Height: | Size: 8.0 KiB |
Before Width: | Height: | Size: 316 KiB After Width: | Height: | Size: 1.7 MiB |
Before Width: | Height: | Size: 8.3 KiB |
Before Width: | Height: | Size: 302 KiB After Width: | Height: | Size: 1.6 MiB |
Before Width: | Height: | Size: 8.7 KiB |
Before Width: | Height: | Size: 349 KiB After Width: | Height: | Size: 1.2 MiB |
Before Width: | Height: | Size: 10 KiB |
Before Width: | Height: | Size: 401 KiB After Width: | Height: | Size: 1.3 MiB |
Before Width: | Height: | Size: 5.0 KiB |
Before Width: | Height: | Size: 247 KiB After Width: | Height: | Size: 1.7 MiB |
Before Width: | Height: | Size: 9.8 KiB |
Before Width: | Height: | Size: 423 KiB After Width: | Height: | Size: 1.8 MiB |
BIN
assets/images/rsapi_diagram_5_alay.png
Normal file
After Width: | Height: | Size: 688 KiB |
BIN
assets/images/rsapimports.png
Normal file
After Width: | Height: | Size: 164 KiB |
Before Width: | Height: | Size: 8.7 KiB |
Before Width: | Height: | Size: 381 KiB After Width: | Height: | Size: 1.4 MiB |
Before Width: | Height: | Size: 13 KiB |
Before Width: | Height: | Size: 484 KiB After Width: | Height: | Size: 1.9 MiB |
BIN
assets/images/srsapi.png.original
Normal file
After Width: | Height: | Size: 1.4 MiB |
Before Width: | Height: | Size: 9.8 KiB |
Before Width: | Height: | Size: 382 KiB After Width: | Height: | Size: 1.9 MiB |
Before Width: | Height: | Size: 11 KiB |
Before Width: | Height: | Size: 364 KiB After Width: | Height: | Size: 1.7 MiB |
Before Width: | Height: | Size: 9.9 KiB |
Before Width: | Height: | Size: 431 KiB After Width: | Height: | Size: 1.7 MiB |
BIN
assets/images/uptime.png
Normal file
After Width: | Height: | Size: 8.4 KiB |
Before Width: | Height: | Size: 10 KiB |
Before Width: | Height: | Size: 403 KiB After Width: | Height: | Size: 1.3 MiB |
Before Width: | Height: | Size: 38 KiB |
Before Width: | Height: | Size: 577 KiB After Width: | Height: | Size: 1.2 MiB |
@ -50,29 +50,24 @@ There are also some other datasets handled differently, because the tools I use
|
||||
|
||||
This is some of what the UI looks like - it is much like a standard Datasette install with a few extra UI elements and some style tweaks I made:
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/maghammer_1.png">
|
||||
<div>Viewing browser history through the table view. This is not great on narrower screens. I'm intending to reengineer this a little at some point.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/maghammer_2.png">
|
||||
<div>The redone search-all interface. My plugin makes clickable links pointing to my media server.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/maghammer_3.png">
|
||||
<div>The front page, listing databases and tables and with the search bar.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/maghammer_1.png
|
||||
Viewing browser history through the table view. This is not great on narrower screens. I'm intending to reengineer this a little at some point.
|
||||
:::
|
||||
::: captioned src=/assets/images/maghammer_2.png
|
||||
The redone search-all interface. My plugin makes clickable links pointing to my media server.
|
||||
:::
|
||||
::: captioned src=/assets/images/maghammer_3.png
|
||||
The front page, listing databases and tables and with the search bar.
|
||||
:::
|
||||
|
||||
Being built out of a tool intended for quantitative data processing means that I can, as I mentioned, do some quantitative data processing. While I could in principle do things like count shell/browser history entries by date, this isn't very interesting, and the cooler datasets are logs from my watch (heart rate and step count), although I haven't gotten around to producing nice aggregates from these, and the manually written structured data entries from my journal. For the reasons described earlier I write up a lot of information in journal entries each day, including machine-readable standardized content. I haven't backfilled this for all entries as it requires a lot of work to read through them and write up the tags, but even with only fairly recent entries usable it's still provided significant insight.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/maghammer_4.png">
|
||||
<div>A simple aggregate query of my notes' structured data. Redacted for privacy.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/maghammer_5.png">
|
||||
<div>Not actually a very helpful format.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/maghammer_4.png
|
||||
A simple aggregate query of my notes' structured data. Redacted for privacy.
|
||||
:::
|
||||
::: captioned src=/assets/images/maghammer_5.png
|
||||
Not actually a very helpful format.
|
||||
:::
|
||||
|
||||
While it's not part of the same system, [Meme Search Engine](https://mse.osmarks.net/) is undoubtedly useful to me for rapidly finding images (memetic images) I need or want - so much so that I have a separate internal instance run on my miscellaneous-images-and-screenshots folder. Nobody else seems to even be trying - while there are a lot of demos of CLIP image search engines on GitHub, and I think one with the OpenAI repository, I'm not aware of *production* implementations with the exception of [clip-retrieval](https://github.com/rom1504/clip-retrieval) and the LAION index deployment, and one iPhone app shipping a distilled CLIP. There's not anything like a user-friendly desktop app, which confuses me somewhat, since there's clearly demand amongst people I talked to. Regardless of the reason, this means that Meme Search Engine is quite possibly the world's most advanced meme search tool (since I bothered to design a nice-to-use query UI and online reindexing), although I feel compelled to mention someone's [somewhat horrifying iPhone OCR cluster](https://findthatmeme.com/blog/2023/01/08/image-stacks-and-iphone-racks-building-an-internet-scale-meme-search-engine-Qzrz7V6T.html). Meme Search Engine is not very well-integrated but I usually know which dataset I want to retrieve from anyway.
|
||||
|
||||
|
@ -21,10 +21,9 @@ As usually happens with sufficiently large projects which I can't neatly complet
|
||||
It did, at least, get away with supporting its capabilities using impressively little code.
|
||||
While wrong people believe that better software involves more code, I, as an enlightened programmer, recognize that you should write as little code as possible, as more code means more bugs *and* more work to write and maintain it.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_1_1.png">
|
||||
<div>Self-replicating Minoteaur pages on the legacy public Minoteaur. I can no longer actually get it to run, so I don't have any other images.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/minoteaur_1_1.png
|
||||
Self-replicating Minoteaur pages on the legacy public Minoteaur. I can no longer actually get it to run, so I don't have any other images.
|
||||
:::
|
||||
|
||||
After deciding that I really did need something which actually worked even if it wasn't perfect, I settled on... installing [DokuWiki](https://www.dokuwiki.org/dokuwiki) - while a PHP application and not particularly modern featurewise, it was known to be robust, supported *most* of what I wanted, and basically worked.
|
||||
I even dabbled in the horrors of PHP to make some tweaks and plugins I wanted work.[^1]
|
||||
@ -34,10 +33,9 @@ Prototypes were developed and reengineered for new, exciting Minoteaurs based on
|
||||
Unfortunately, despite the nicer interface and actually-useful Markdown preview mode, these proved frustrating to work on, due to the usual difficulties of maintaining consistency between client code with persistent state and a server which also has persistent state, and I ultimately consigned Minoteaur 2 and 3 to the depths of my project folder in mid-2020.
|
||||
It was also somewhat slow, due to the overhead of parsing Markdown into a parse tree and then rendering that parse tree to virtual DOM and then rendering virtual DOM to actual DOM nodes.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_2.png">
|
||||
<div>Minoteaur 2's multitasking UI was ultimately abandoned in favour of just using native browser tabs or windows.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/minoteaur_2.png
|
||||
Minoteaur 2's multitasking UI was ultimately abandoned in favour of just using native browser tabs or windows.
|
||||
:::
|
||||
|
||||
While researching for this I thought that Minoteaur 4 didn't actually exist because of accidents with the numbering scheme, but it turns out that it does, but it's actually Minoteaur 1.5.
|
||||
I had vague memories of a prototype Rust backend for use with a single-page application (with a vaguely RESTful API rather than the usual tightly-coupled RPC designs I use) which I had assumed was developed after the Node.js ones, but it was actually made significantly before those in late 2019.
|
||||
@ -54,18 +52,15 @@ Nim is sort of how I would design a programming language, both in the sense that
|
||||
It has enough working libraries for things like SQLite and webservers that I thought it worth trying anyway, and it was indeed the most functional Minoteaur at the time, incorporating good SQLite-based search, backlinks, a mostly functional UI, partly style-insensitive links, a reasonably robust parser, a decent UI, and even DokuWiki-like drafts in the editor (a feature I end up using quite often due to things like accidentally closing or refreshing pages).
|
||||
However, I got annoyed again by the server-rendered design, the terrible, terrible code I had to write to directly bind to a C-based GFM library (I think I at least managed to make it not segfault, even though I don't know why), and probably some things I forgot, leading to the *next* version.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_6.png">
|
||||
<div>The Minoteaur 6 "two-pane" editor UI.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_6_2.png">
|
||||
<div>Its search mechanism worked, but with some UI problems.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_6_3.png">
|
||||
<div>Minoteaur 6 had an extensive login fallback system.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/minoteaur_6.png
|
||||
The Minoteaur 6 "two-pane" editor UI.
|
||||
:::
|
||||
::: captioned src=/assets/images/minoteaur_6_2.png
|
||||
Its search mechanism worked, but with some UI problems.
|
||||
:::
|
||||
::: captioned src=/assets/images/minoteaur_6_3.png
|
||||
Minoteaur 6 had an extensive login fallback system.
|
||||
:::
|
||||
|
||||
Python is my go-to language for rapid prototyping, i.e. writing poor-quality code very quickly, so it made some sense for me to rewrite in that next in 2021.
|
||||
Minoteaur 7 was a short-lived variant using server rendering, which was rapidly replaced by Minoteaur 7.1, which used a frontend web framework called Svelte for its UI[^3].
|
||||
@ -77,14 +72,12 @@ This was actually quite easy to do thanks to the hard work of library developers
|
||||
It also got the furthest yet in terms of general usability, mostly because I implemented file upload, which I think is necessary for any useful notes software (you do, at the very least, want to be able to add and reference images).
|
||||
Ultimately, for some reason I forgot (I think mostly database management this time), I decided that I disliked the code and rewrote it yet again, leading to Minoteaur 8.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_7.png">
|
||||
<div>Minoteaur 7.1 introduced a new UI style and more effectively used the width of standard horizontal screens with the search sidebar. It also incorporated page icons for the first time.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_7_1.png">
|
||||
<div>An earlier prototype displaying its search capabilities.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/minoteaur_7.png
|
||||
Minoteaur 7.1 introduced a new UI style and more effectively used the width of standard horizontal screens with the search sidebar. It also incorporated page icons for the first time.
|
||||
:::
|
||||
::: captioned src=/assets/images/minoteaur_7_1.png
|
||||
An earlier prototype displaying its search capabilities.
|
||||
:::
|
||||
|
||||
[Minoteaur 8](https://github.com/osmarks/minoteaur-8) was a rewrite in Rust again, starting in February 2022, using much of the frontend code from Minoteaur 7.1 but with a completely different backend but with a similar architecture, apart from the fact that instead of using SQLite directly and "sensibly", it uses it to store persistent versions of objects (revisions, pageviews or pages) for which live copies and indices are held in memory.
|
||||
Since notes aren't really *that* big, I worked out that even under pessimistic assumptions the RAM requirements would be lower than those of the JS/Python interpreter processes running previous Minoteaurs, which were not particularly large anyway (more on this in my future writing on how all software is terrible), and this made a lot of the code simpler due to not having to limit data structures to what SQLite supports and not having to deal with async IO for read operations.
|
||||
@ -92,14 +85,12 @@ Since notes aren't really *that* big, I worked out that even under pessimistic a
|
||||
Despite considerable success in making it work to the same extent as previous Minoteaurs (files, search, backlinks, Markdown, etc) and even somewhat further (nicer Markdown syntax, and a three-pane UI), development was mysteriously halted for a while in March and nonmysteriously (some inconsistencies in how context for backlinks versus for search worked which felt annoying to fix) in July after I picked it back up.
|
||||
This April, I happened to look again for some reason, and found that the problem was actually easy if reframed slightly, then did everything else I wanted for usability parity with my DokuWiki install over the course of three days, wrote an import and DokuWiki migration script, redid some of the syntax for more reliable parsing, and finally transitioned away from DokuWiki after slightly less than 4 years.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_8.png">
|
||||
<div>Minoteaur's Minoteaur page.</div>
|
||||
</div>
|
||||
<div class="caption">
|
||||
<img src="/assets/images/minoteaur_8_0.png">
|
||||
<div>To allow discovery of interesting content you may have forgotten, Minoteaur incorporates a random pages list.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/minoteaur_8.png
|
||||
Minoteaur's Minoteaur page.
|
||||
:::
|
||||
::: captioned src=/assets/images/minoteaur_8_0.png
|
||||
To allow discovery of interesting content you may have forgotten, Minoteaur incorporates a random pages list.
|
||||
:::
|
||||
|
||||
## So what does it actually do?
|
||||
|
||||
|
@ -37,10 +37,9 @@ Intel GPUs have good matrix multiplication accelerators, but their most powerful
|
||||
|
||||
Many unwary buyers have fallen for the siren song of increasingly cheap used Nvidia Tesla GPUs, since they offer very large VRAM pools at very low cost. However, these are a bad choice unless you *only* need that VRAM. The popular Tesla K80 is 9 years old, with lacking driver support, no FP16, extremely lacking general performance, high power consumption, and no modern optimization efforts, and it's not actually one GPU - it's two on a single card, so you have to deal with parallelizing anything big across GPUs. The next-generation Tesla M40 has similar problems, although it is a single GPU rather than two, and the P40 is not much different, though instead of *no* FP16 it has *unusably slow* FP16[^14]. Even a Tesla P100 is lacking in compute performance compared to newer generations. Datacentre cards newer than that are not available cheaply. There's also some complexity with cooling, since they're designed for server airflow with separate fans, unlike a consumer GPU.[^13]
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/tesla-k80.jpg">
|
||||
<div>It may look innocent, but it is a menace to unaware hobbyists.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/tesla-k80.jpg
|
||||
It may look innocent, but it is a menace to unaware hobbyists.
|
||||
:::
|
||||
|
||||
### Do not buy workstation cards
|
||||
|
||||
@ -72,10 +71,9 @@ GPUs are pretty power-hungry. PCPartPicker will make a good estimate of maximum
|
||||
|
||||
If you're concerned about reducing your power bill, Ada Lovelace GPUs are generally much more efficient than Ampere due to their newer manufacturing process. You can also power-limit your GPU using `nvidia-smi -pl [power limit in watts]` (note that this must be run each boot in some way): this does reduce performance, but nonlinearly.
|
||||
|
||||
<div class="caption">
|
||||
<img src="/assets/images/rtx-4090-power-scaling.webp">
|
||||
<div>Thanks to "snowy χατγιρλ/acc" on #off-topic for the benchmark. Other GPUs will have different behaviour. This is something of a worst case though - you'll lose less to power limits in real workloads.</div>
|
||||
</div>
|
||||
::: captioned src=/assets/images/rtx-4090-power-scaling.webp
|
||||
Thanks to "snowy χατγιρλ/acc" on #off-topic for the benchmark. Other GPUs will have different behaviour. This is something of a worst case though - you'll lose less to power limits in real workloads.
|
||||
:::
|
||||
|
||||
## Other components
|
||||
|
||||
|
101
blog/stack-rsapi.md
Normal file
@ -0,0 +1,101 @@
|
||||
---
|
||||
title: "Site tech stack 2: the unfathomed depths"
|
||||
description: RSAPI and the rest of my infrastructure.
|
||||
created: 27/03/2024
|
||||
slug: srsapi
|
||||
---
|
||||
The original [Site tech stack](/stack/) article (updated since release somewhat as hardware has improved and software been replaced) covers the basic workings of the public-facing website. However, I run *other* things, some of which are interesting to talk about! I have a number of services for personal use running on the same infrastructure, and several non-web-facing but public services. Here's the latest edition of the handy diagram I made in Graphviz:
|
||||
|
||||
::: captioned src=/assets/images/rsapi_diagram_5_alay.png wide link
|
||||
This used to be done manually in draw.io, before it became intractable to run the layouts by hand. You may want to open it in fullscreen.
|
||||
:::
|
||||
|
||||
This is split into several boxes indicating the various servers several subsystems run on. As I mention in the comments of the old article, I have a physical server running the actual compute tasks (`protagonism`), but a <span class="hoverdefn" title="Virtual Private Server (cloud VM)">VPS</span> (`procyon`) is where your HTTP requests are initially going. Since it tends to have better uptime[^1], it also runs the [uptime monitoring system](https://status.osmarks.net/) onstat3, my [Discord bot](https://github.com/osmarks/autobotrobot), and a few other directly network-facing things: osmarksDNS[^3], an <span class="hoverdefn" title="Internet Relay Chat (the Discord of the 1990s)">IRC</span> server ([APIONET](https://apionet.gh0.pw/)) and smtp2rss.
|
||||
|
||||
If you know what <span class="hoverdefn" title="Simple Mail Transfer Protocol">SMTP</span> stands for, hearing "smtp2rss" may have confused you (if you have not talked to me much) or worried you (if you have). Don't worry: it's perfectly sane and reasonable. I [like RSS](/rssgood/), but many people try to email me things, without an RSS fallback[^2]. I dislike minor workflow inconveniences and am willing to throw arbitrary amounts of technology and engineering at them (sometimes), so I wrote a [Python script](https://github.com/osmarks/random-stuff/blob/master/smtp2rss.py) to take inbound emails on a spare domain and expose them as RSS feeds. As a handy bonus, it can provide disposable mailboxes for signing up to services.
|
||||
|
||||
A fun quirk of the nginx installation on `procyon` is that, since I wanted it to not be able to decrypt requests to `protagonism` (I don't entirely trust it, and duplicating the certificate issuance programs on each would be irritating), I use [ngx_stream_ssl_preread](https://nginx.org/en/docs/stream/ngx_stream_ssl_preread_module.html) to forward still-encrypted TLS connections either to itself (on another port) or `protagonism`'s reverse proxy. As janky as this sounds, it does seem to work fine, except for one extremely-hard-to-reproduce bug I suspect might be related where users sometimes get shown 404 pages or the status page incorrectly. Traffic is routed over [Tailscale](https://tailscale.com/) using [Headscale](https://github.com/juanfont/headscale)[^5].
|
||||
|
||||
Several of `protagonism`'s services are mostly-self-contained personal-use applications, such as [Minoteaur](/minoteaur/) (notes), [ankisyncd](https://github.com/ankicommunity/ankicommunity-sync-server/) (flashcards), [atuin](https://github.com/atuinsh/atuin) (shell history) and [calibre-web](https://github.com/janeczku/calibre-web) (books). The rest are somewhat more interesting, in that they do more and are in some cases publicly accessible. For example, [SPUDNET](https://d.gh0.pw/doku.php?id=gtech:spudnet). It was built to serve the needs of an ["operating system"](https://potatos.madefor.cc/) for ComputerCraft (a Minecraft computer mod) by providing <span class="hoverdefn" title="backdoors">remote debugging services</span>. Originally built about six years ago, it somehow still works with relatively minor changes (new protocol support). It provides bidirectional many-to-one and many-to-many communications over websocket, with an unnecessarily sophisticated authentication system, as well as HTTP long polling fallbacks and incident reports. [Skynet](https://github.com/osmarks/skynet) is a somewhat simpler version.
|
||||
|
||||
I also have a monitoring system using [VictoriaMetrics](https://docs.victoriametrics.com/)[^4] and [Grafana](https://grafana.com/). VictoriaMetrics periodically scrapes services for metrics and stores time series, and Grafana can plot them. This is a fairly standard setup, and lots of software exposes Prometheus-compatible metrics itself or has an exporter available (e.g. the [node_exporter](https://github.com/prometheus/node_exporter) for general Linux machine status and the [PostgreSQL exporter](https://github.com/prometheus-community/postgres_exporter)). I went slightly further by exposing metrics in most of my custom applications, so I have, for instance, nice dashboards from my Discord bot. These used to be public, but apparently that exposing any dashboard in Grafana allows users to read any data out of the backend, which was a bit of a security issue. One might reasonably question how much use I get out of these, as I don't get enough traffic to have to debug performance issues much, but they do look nice and their presence is calming.
|
||||
|
||||
::: captioned src=/assets/images/grafana1.png
|
||||
My customized node-exporter dashboard. I still need more storage.
|
||||
:::
|
||||
::: captioned src=/assets/images/grafana2.png
|
||||
AutoBotRobot monitoring.
|
||||
:::
|
||||
|
||||
The most important component I have is undoubtedly RSAPI, the highly custom integration script which does about twenty different things since putting them in different services would have been annoying. Many things I need to do share the same basic building blocks - a database or simple state storage, timers and an HTTP server - and while it would be *possible* though tricky to factor this out into a library and write several microservices, the deployment would be harder to manage with my tools and many of the parts have to interoperate anyway. Some would consider a 1600-line monolithic Python program plugged into 10 different APIs "bad", but, freed from Conway's law[^6], I think it is actually the most efficient way to do this. For whatever reason I can track its structure mentally very efficiently, so it's not hard to work on.
|
||||
|
||||
::: captioned src=/assets/images/rsapimports.png
|
||||
The RSAPI imports section.
|
||||
:::
|
||||
|
||||
RSAPI has a wide range of functions, having grown from a short Flask application which served [fortunes](https://wiki.archlinux.org/title/Fortune) to [PotatOS](https://potatos.madefor.cc/) by accretion of additional capabilities as they were needed. The exact history has been lost to the halcyon days of poor version control and backups, but it was built in roughly this order:
|
||||
|
||||
* Initial version built: served fortunes and [https://schlockmercenary.fandom.com/wiki/The_Seventy_Maxims_of_Maximally_Effective_Mercenaries](https://schlockmercenary.fandom.com/wiki/The_Seventy_Maxims_of_Maximally_Effective_Mercenaries) over HTTP.
|
||||
* "Currently playing" support for my internet radio server, via integration with MPD.
|
||||
* youtube-dl web frontend and very basic login.
|
||||
* IRC bot for server status and MPD status.
|
||||
* Rewrite from Flask/gevent to asyncio/aiohttp.
|
||||
* DNS to comments (and IRC) bridge - converts specially formatted DNS queries to a subdomain to comments on a certain page.
|
||||
* "Currently playing" support expanded with listener counts.
|
||||
* [Miniflux](https://miniflux.app/) (RSS reader) to Discord bridge, and (unnecessary, since it has an RSS button I somehow ignored) RoyalRoad fiction to RSS bridge.
|
||||
* Random video selector (from local media folders).
|
||||
* [IncDec](https://osmarks.net/incdec/)/IRC Bridge.
|
||||
* Cancelled rewrite to make it more modular. Migration of internal databases from SQLite to LevelDB.
|
||||
* Live chat on internet radio (specialized bridge to IRC).
|
||||
* NASA [Astronomy Picture of the Day](https://apod.nasa.gov/apod/) fetcher implemented. Daily task scheduler built.
|
||||
* Wordnik Word of the Day fetcher implemented.
|
||||
* ComputerCraft to Prometheus metrics bridge.
|
||||
* "[Webmaze](https://r.osmarks.net/maze/ZSwLxemYUq59J-Hcr-rx0ejcJmMvrjAhA4Nxa7KcBgiNmmTVa8ZxDHVw-ZVZXhFMxj_6kA)" prototype - an infinite partly connected 3D grid implemented fully statelessly, intended for a hypothetical adventure game which never materialized.
|
||||
* [Freefall](http://freefall.purrsia.com/) (webcomic) fetcher implemented.
|
||||
* Migration from youtube-dl to yt_dlp.
|
||||
* Cross-device scratchpad/clipboard for copying short text between computers.
|
||||
* For some reason, a script which infinitely generates primes, digits of e and digits of pi, reads them out using bad TTS, and sends them as an internet radio stream[^9].
|
||||
* Random bytes API.
|
||||
* Basic service status page for a tablet (to be glued to walls).
|
||||
* Migration back to SQLite. Proper database migration system.
|
||||
* URL shortener - in conjunction with my [really short domain](https://0t.lt/), it can produce really short URLs, as well as ZWS-based URLs and two-word URLs like [https://0t.lt/YearningPried](https://0t.lt/YearningPried) - these use a special prefix-free wordlist so they can also be typed in uncapitalized unambiguously.
|
||||
* [zzcxz](https://zzcxz.citrons.xyz/) telephony interface.
|
||||
* Tablet status interface enhanced with list of failed services (from systemd).
|
||||
* Personal event logger.
|
||||
* Remote to local calendar synchronizer.
|
||||
* Weather and calendar updates (for the tablet).
|
||||
* [Threat Update (Twitter)](https://twitter.com/threat_update) scraper.
|
||||
* [SwitchCraft](https://sc3.io/) player surveillance (Dynmap).
|
||||
* RSS feed for random memes (for XScreensaver).
|
||||
* Better login system - multiple users, SSO for other services via [Nginx authentication subrequests](https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/), basicauth option for non-interactive systems.
|
||||
* Key/value storage backend for PotatOS, due to the shutdown of the random free API it used before.
|
||||
* Internal LLM-based [Threat Updates](https://r.osmarks.net/threat-update) system[^8], to replace the archive of historical ones and Twitter scraper. I was too lazy to work out how to draw nicely line-wrapped text in images in Python, so this actually invokes a ComputerCraft emulator, runs the Threat Update implementation on that, dumps its virtual screen, and renders that to an image.
|
||||
* The new comments system, replacing [Isso](https://github.com/isso-comments/isso). It supports ominous AI faces (from StyleGAN2, thanks to [StyleGANCpp](https://github.com/podgorskiy/StyleGANCpp/)[^7]), leftvotes/rightvotes for greater user expression, SSO integration, better threading, and lower client resource use.
|
||||
|
||||
I'm especially proud of the ComputerCraft to Prometheus metrics bridge. While it's only about 20 lines of code (plus the ComputerCraft side), it does allow me to feel cool about being able to monitor meaninglessly large numbers in great detail. There are similar Factorio mods, which I'll probably use next time I play.
|
||||
|
||||
::: captioned src=/assets/images/grafana3.png
|
||||
My base's reactor powering up on a recent tech modpack server, as visualized from Grafana.
|
||||
:::
|
||||
|
||||
I also have some custom inference servers backing [Meme Search Engine](https://mse.osmarks.net/) and [Maghammer](/maghammer/), in addition to an ExllamaV2-based LLM API used in PotatOS. Early prototypes loaded the models in-process, but this was very inflexible: restarts were slow, only one process at a time could use them, and it effectively required that consuming code be written in Python. The servers are basic (no automatic batching and few optimizations), but are presently good enough to handle traffic. The CLIP one is in fact open as part of [Meme Search Engine](https://github.com/osmarks/meme-search-engine/blob/master/clip_server.py).
|
||||
|
||||
I haven't covered *every* osmarks.net service in this post, or even all the ones in the slightly outdated diagram above, but I think I got the most interesting ones. I hope this was informative, and did not accidentally make people notice horrible security issues I missed.
|
||||
|
||||
[^1]: Much better uptime: ![An SSH session on procyon saying "up 588 days, 38 min".](/assets/images/uptime.png)
|
||||
|
||||
[^2]: Ironically, they are doing this via mailing list services which absolutely could also offer RSS if they wanted to (Mailchimp does, even, as an option). They probably don't want to for "engagement" reasons.
|
||||
|
||||
[^3]: osmarksDNS is less interesting, and refers to a DNS over HTTPS server and recursive resolver installed locally, I think because of an issue with bootstrapping dnscrypt-proxy I had years ago. That was fixed another way, but I never had a compelling reason to shut it down.
|
||||
|
||||
[^4]: This used to be Prometheus, but I swapped VictoriaMetrics in to reduce storage requirements.
|
||||
|
||||
[^5]: I like Tailscale's ease of use, but it's horrifyingly CPU-intensive for no obvious reason, and `procyon` is not very powerful. This would be a problem if I had traffic.
|
||||
|
||||
[^6]: "The structure of any system designed by an organization is isomorphic to the structure of the organization." You could argue that this is more "directly in line with Conway's law" than "freed from it", but ignore that.
|
||||
|
||||
[^7]: It seems to have a gender bias problem, presumably due to dataset or potentially broken implementation. I wanted to use Stable Diffusion, but the compute costs are too bad to run it on CPU and I don't have the free VRAM to load it on GPU constantly.
|
||||
|
||||
[^8]: Here's the latest: <img src="https://r.osmarks.net/threat-update">
|
||||
|
||||
[^9]: That one bothers me. It contains an algorithm for streaming digits of π which I clearly got from GitHub somewhere, but the one for e is written in my style but isn't something I understand or recognize.
|
@ -12,12 +12,12 @@ The main site itself, which you're currently reading, is in fact just a simple s
|
||||
|
||||
Being static files, many, many different webservers could have been used for this site. In practice, it's mostly alternated randomly between [caddy](https://caddyserver.com/) (a more recent, Go-based webserver with automatic LetsEncrypt integration) and [nginx](https://nginx.org/) (an older and more powerful but slightly quirky program) - caddy generally had easier configuration, but I arbitrarily preferred nginx in some ways. After caddy v2 suddenly required me to rewrite my configuration and introduced a bunch of weird issues, I permanently switched over to nginx and haven't changed back. The configuration file is now 600 lines or so, even with inclusion of includes to shorten things, but it... works, at least. This is mostly to accommodate the bizzarely large set of subdomains I now have for various people, and reverse proxy configuration for backend services. I also use a custom-compiled build of nginx with HTTP/3 (QUIC) support and some modules compiled in.
|
||||
|
||||
Some of these backend things are only for personal use, but a few are related to the site itself. For example, the comment server is a standalone Python program, [isso](https://posativ.org/isso/), with corresponding JS embedded in each page. This works pretty well, but has lead to some weird quirkiness, such as each separate 404-erroring URL having its own list of comments. There's also the Random Stuff API, a custom assemblage of about 15 different Python libraries and external programs which, while technically not linked on the site, does interact with other projects like [PotatOS](https://git.osmarks.net/osmarks/potatOS/), and internal services on the same infrastructure like my [RSS reader](https://miniflux.app/). The images subdomain also uses a [PHP program](https://larsjung.de/h5ai/) to generate a nice searchable index; in fact, it is <del>one of two</del> the only PHP thing<del>s</del> I have unfortunately not yet been able to purge[^1]. There also used to be a publicly available status page using some custom code, but this doesn't work very well and has now been dropped; previously I had a Grafana (and earlier Netdata) instance there, but this has now been cancelled because it leaks a worrying amount of information.
|
||||
Some of these backend things are only for personal use, but a few are related to the site itself. For example, the comment server is ~~a standalone Python program, [isso](https://posativ.org/isso/),~~ now part of [RSAPI](/srsapi/) with corresponding JS embedded in each page. This works pretty well, but has lead to some weird quirkiness, such as each separate 404-erroring URL having its own list of comments. There's also the Random Stuff API, a custom assemblage of about 15 different Python libraries and external programs which, while technically not linked on the site, does interact with other projects like [PotatOS](https://git.osmarks.net/osmarks/potatOS/), and internal services on the same infrastructure like my [RSS reader](https://miniflux.app/). The images subdomain also uses a [PHP program](https://larsjung.de/h5ai/) to generate a nice searchable index; in fact, it is <del>one of two</del> the only PHP thing<del>s</del> I have unfortunately not yet been able to purge[^1]. There also used to be a publicly available status page using some custom code, but this doesn't work very well and has now been dropped; previously I had a Grafana (and earlier Netdata) instance there, but this has now been cancelled because it leaks a worrying amount of information.
|
||||
|
||||
As for the underlying OS everything runs on, I currently use [Arch Linux](https://i.osmarks.net/memes-or-something/arch-btw.png) (as well as Alpine on a few lower-resourced cloud servers). Some form of Linux is inevitable - BSDs aren't really compatible with much, and Windows is obviously unsuited for server duty - but I mostly use Arch for its stability (this sounds sarcastic, but I've actually found it to be very reliable with regular updates), wide range of packages (particularly from the AUR; as I don't really run critical production infrastructure, I can generally afford to compile stuff from source a lot), and better general ease-of-use than Alpine. As much as I vaguely resent it, this is mostly down to systemd - despite it being a horrific bloated monolith, `journalctl` is very convenient and unit files are pleasant and easy to write compared to the weird OpenRC scripts Alpine uses.
|
||||
|
||||
I am actually considering yet another redesign, however; switching to a dynamic site implementation instead would allow me to integrate the comment system and achievement system better, make things like the "from other blogs" tiles actually update at reasonable intervals, and arbitrarily A/B test users, although it would break some nice things like this site's very aggressive caching and fast serving. Please leave your thoughts or lack of thoughts on this in the comments.
|
||||
|
||||
[^1]: The previous one was DokuWiki, now replaced with Minoteaur.
|
||||
[^1]: The previous one was DokuWiki, now replaced with [Minoteaur](/minoteaur/).
|
||||
|
||||
[^2]: My next upgrade is probably going to be more SSD space, since I'm *somehow* running out of that.
|
110
package-lock.json
generated
@ -20,8 +20,9 @@
|
||||
"handlebars": "^4.7.6",
|
||||
"html-minifier": "^4.0.0",
|
||||
"idb": "^7.1.1",
|
||||
"markdown-it": "^13.0.1",
|
||||
"markdown-it": "^14.1.0",
|
||||
"markdown-it-anchor": "^8.6.7",
|
||||
"markdown-it-container": "^4.0.0",
|
||||
"markdown-it-footnote": "^3.0.3",
|
||||
"mustache": "^4.0.1",
|
||||
"nanoid": "^2.1.11",
|
||||
@ -777,9 +778,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/entities": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz",
|
||||
"integrity": "sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==",
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
|
||||
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
|
||||
"engines": {
|
||||
"node": ">=0.12"
|
||||
},
|
||||
@ -1237,11 +1238,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/linkify-it": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-4.0.1.tgz",
|
||||
"integrity": "sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==",
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
|
||||
"integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
|
||||
"dependencies": {
|
||||
"uc.micro": "^1.0.1"
|
||||
"uc.micro": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/lower-case": {
|
||||
@ -1261,18 +1262,19 @@
|
||||
}
|
||||
},
|
||||
"node_modules/markdown-it": {
|
||||
"version": "13.0.1",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-13.0.1.tgz",
|
||||
"integrity": "sha512-lTlxriVoy2criHP0JKRhO2VDG9c2ypWCsT237eDiLqi09rmbKoUetyGHq2uOIRoRS//kfoJckS0eUzzkDR+k2Q==",
|
||||
"version": "14.1.0",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
|
||||
"integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
|
||||
"dependencies": {
|
||||
"argparse": "^2.0.1",
|
||||
"entities": "~3.0.1",
|
||||
"linkify-it": "^4.0.1",
|
||||
"mdurl": "^1.0.1",
|
||||
"uc.micro": "^1.0.5"
|
||||
"entities": "^4.4.0",
|
||||
"linkify-it": "^5.0.0",
|
||||
"mdurl": "^2.0.0",
|
||||
"punycode.js": "^2.3.1",
|
||||
"uc.micro": "^2.1.0"
|
||||
},
|
||||
"bin": {
|
||||
"markdown-it": "bin/markdown-it.js"
|
||||
"markdown-it": "bin/markdown-it.mjs"
|
||||
}
|
||||
},
|
||||
"node_modules/markdown-it-anchor": {
|
||||
@ -1284,6 +1286,11 @@
|
||||
"markdown-it": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/markdown-it-container": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it-container/-/markdown-it-container-4.0.0.tgz",
|
||||
"integrity": "sha512-HaNccxUH0l7BNGYbFbjmGpf5aLHAMTinqRZQAEQbMr2cdD3z91Q6kIo1oUn1CQndkT03jat6ckrdRYuwwqLlQw=="
|
||||
},
|
||||
"node_modules/markdown-it-footnote": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz",
|
||||
@ -1295,9 +1302,9 @@
|
||||
"integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
|
||||
},
|
||||
"node_modules/mdurl": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
|
||||
"integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4="
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
|
||||
"integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w=="
|
||||
},
|
||||
"node_modules/mime-db": {
|
||||
"version": "1.52.0",
|
||||
@ -1591,6 +1598,14 @@
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/punycode.js": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
|
||||
"integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/ramda": {
|
||||
"version": "0.26.1",
|
||||
"resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz",
|
||||
@ -1901,9 +1916,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/uc.micro": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz",
|
||||
"integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA=="
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
|
||||
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="
|
||||
},
|
||||
"node_modules/uglify-js": {
|
||||
"version": "3.13.10",
|
||||
@ -2429,9 +2444,9 @@
|
||||
}
|
||||
},
|
||||
"entities": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz",
|
||||
"integrity": "sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q=="
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
|
||||
"integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="
|
||||
},
|
||||
"esbuild": {
|
||||
"version": "0.19.6",
|
||||
@ -2751,11 +2766,11 @@
|
||||
"integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw=="
|
||||
},
|
||||
"linkify-it": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-4.0.1.tgz",
|
||||
"integrity": "sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==",
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz",
|
||||
"integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==",
|
||||
"requires": {
|
||||
"uc.micro": "^1.0.1"
|
||||
"uc.micro": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"lower-case": {
|
||||
@ -2772,15 +2787,16 @@
|
||||
}
|
||||
},
|
||||
"markdown-it": {
|
||||
"version": "13.0.1",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-13.0.1.tgz",
|
||||
"integrity": "sha512-lTlxriVoy2criHP0JKRhO2VDG9c2ypWCsT237eDiLqi09rmbKoUetyGHq2uOIRoRS//kfoJckS0eUzzkDR+k2Q==",
|
||||
"version": "14.1.0",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz",
|
||||
"integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==",
|
||||
"requires": {
|
||||
"argparse": "^2.0.1",
|
||||
"entities": "~3.0.1",
|
||||
"linkify-it": "^4.0.1",
|
||||
"mdurl": "^1.0.1",
|
||||
"uc.micro": "^1.0.5"
|
||||
"entities": "^4.4.0",
|
||||
"linkify-it": "^5.0.0",
|
||||
"mdurl": "^2.0.0",
|
||||
"punycode.js": "^2.3.1",
|
||||
"uc.micro": "^2.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"argparse": {
|
||||
@ -2796,15 +2812,20 @@
|
||||
"integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==",
|
||||
"requires": {}
|
||||
},
|
||||
"markdown-it-container": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it-container/-/markdown-it-container-4.0.0.tgz",
|
||||
"integrity": "sha512-HaNccxUH0l7BNGYbFbjmGpf5aLHAMTinqRZQAEQbMr2cdD3z91Q6kIo1oUn1CQndkT03jat6ckrdRYuwwqLlQw=="
|
||||
},
|
||||
"markdown-it-footnote": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz",
|
||||
"integrity": "sha512-YZMSuCGVZAjzKMn+xqIco9d1cLGxbELHZ9do/TSYVzraooV8ypsppKNmUJ0fVH5ljkCInQAtFpm8Rb3eXSrt5w=="
|
||||
},
|
||||
"mdurl": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz",
|
||||
"integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4="
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz",
|
||||
"integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w=="
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.52.0",
|
||||
@ -3059,6 +3080,11 @@
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"punycode.js": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz",
|
||||
"integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA=="
|
||||
},
|
||||
"ramda": {
|
||||
"version": "0.26.1",
|
||||
"resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz",
|
||||
@ -3264,9 +3290,9 @@
|
||||
}
|
||||
},
|
||||
"uc.micro": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz",
|
||||
"integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA=="
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz",
|
||||
"integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A=="
|
||||
},
|
||||
"uglify-js": {
|
||||
"version": "3.13.10",
|
||||
|
@ -15,8 +15,9 @@
|
||||
"handlebars": "^4.7.6",
|
||||
"html-minifier": "^4.0.0",
|
||||
"idb": "^7.1.1",
|
||||
"markdown-it": "^13.0.1",
|
||||
"markdown-it": "^14.1.0",
|
||||
"markdown-it-anchor": "^8.6.7",
|
||||
"markdown-it-container": "^4.0.0",
|
||||
"markdown-it-footnote": "^3.0.3",
|
||||
"mustache": "^4.0.1",
|
||||
"nanoid": "^2.1.11",
|
||||
|
5
src/avif_compact.sh
Executable file
@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
file=$(mktemp /tmp/tmp.XXXXXXXXXX.png)
|
||||
convert "$1" -resize 25% "$file"
|
||||
avifenc -s 0 -q 50 "$file" $2
|
||||
rm "$file"
|
@ -30,7 +30,7 @@
|
||||
"https://www.rtl-sdr.com/feed/",
|
||||
"https://astralcodexten.substack.com/feed",
|
||||
"https://www.rifters.com/crawl/?feed=rss2",
|
||||
"https://drewdevault.com/feed.xml",
|
||||
"https://drewdevault.com/blog/index.xml",
|
||||
"https://qntm.org/rss.php",
|
||||
"https://aphyr.com/posts.atom",
|
||||
"https://os.phil-opp.com/rss.xml"
|
||||
|
47
src/index.js
@ -76,9 +76,42 @@ globalData.hashBG = hashBG
|
||||
|
||||
const removeExtension = x => x.replace(/\.[^/.]+$/, "")
|
||||
|
||||
const mdutils = MarkdownIt().utils
|
||||
const renderContainer = (tokens, idx) => {
|
||||
let opening = true
|
||||
if (tokens[idx].type === "container__close") {
|
||||
let nesting = 0
|
||||
for (; tokens[idx].type !== "container__open" && nesting !== 1; idx--) {
|
||||
nesting += tokens[idx].nesting
|
||||
}
|
||||
opening = false
|
||||
}
|
||||
const m = tokens[idx].info.trim().split(" ");
|
||||
const blockType = m[0]
|
||||
|
||||
const options = {}
|
||||
for (const arg of m.slice(1)) {
|
||||
const [k, v] = arg.split("=", 2)
|
||||
options[k] = v ?? true
|
||||
}
|
||||
|
||||
if (opening) {
|
||||
if (blockType === "captioned") {
|
||||
const link = `<a href="${md.utils.escapeHtml(options.src)}">`
|
||||
return `<div class="${options.wide ? "caption wider" : "caption"}">${options.link ? link : ""}<img src="${md.utils.escapeHtml(options.src)}">${options.link ? "</a>" : ""}`
|
||||
}
|
||||
} else {
|
||||
if (blockType === "captioned") {
|
||||
return `</div>`
|
||||
}
|
||||
}
|
||||
throw new Error(`unrecognized blockType ${blockType}`)
|
||||
}
|
||||
|
||||
const readFile = path => fsp.readFile(path, { encoding: "utf8" })
|
||||
const anchor = require("markdown-it-anchor")
|
||||
const md = new MarkdownIt({ html: true })
|
||||
.use(require("markdown-it-container"), "", { render: renderContainer, validate: params => true })
|
||||
.use(require("markdown-it-footnote"))
|
||||
.use(anchor, {
|
||||
permalink: anchor.permalink["headerLink"]({
|
||||
@ -332,10 +365,9 @@ const doImages = async () => {
|
||||
(await fse.readdir(path.join(assetsDir, "images"), { encoding: "utf-8" })).map(async image => {
|
||||
if (image.endsWith(".original")) { // generate alternative formats
|
||||
const stripped = image.replace(/\.original$/).split(".").slice(0, -1).join(".")
|
||||
globalData.images[stripped] = {}
|
||||
const fullPath = path.join(assetsDir, "images", image)
|
||||
const stat = await fse.stat(fullPath)
|
||||
const writeFormat = async (name, ext, mime, cmd, supplementaryArgs) => {
|
||||
const writeFormat = async (name, ext, cmd, supplementaryArgs, suffix="") => {
|
||||
let bytes = readCache(`images/${stripped}/${name}`, null, stat.mtimeMs)
|
||||
const destFilename = stripped + ext
|
||||
const destPath = path.join(outAssets, "images", destFilename)
|
||||
@ -350,10 +382,15 @@ const doImages = async () => {
|
||||
await fsp.writeFile(destPath, bytes)
|
||||
}
|
||||
|
||||
globalData.images[stripped][mime] = "/assets/images/" + destFilename
|
||||
return "/assets/images/" + destFilename
|
||||
}
|
||||
await writeFormat("avif", ".avif", "image/avif", "avifenc", ["-s", "0", "-q", "20"])
|
||||
await writeFormat("jpeg-scaled", ".jpg", "_fallback", "convert", ["-resize", "25%", "-format", "jpeg"])
|
||||
const avif = await writeFormat("avif", ".avif", "avifenc", ["-s", "0", "-q", "20"], " 2x")
|
||||
const avifc = await writeFormat("avif-compact", ".c.avif", path.join(srcDir, "avif_compact.sh"), [])
|
||||
const jpeg = await writeFormat("jpeg-scaled", ".jpg", "_fallback", "convert", ["-resize", "25%", "-format", "jpeg"])
|
||||
globalData.images[stripped] = [
|
||||
["image/avif", `${avifc}, ${avif} 2x`],
|
||||
["_fallback", jpeg]
|
||||
]
|
||||
} else {
|
||||
globalData.images[image.split(".").slice(0, -1).join(".")] = "/assets/images/" + image
|
||||
}
|
||||
|
@ -417,7 +417,7 @@ if (sidenotes && footnotes) {
|
||||
const linkRect = link.getBoundingClientRect()
|
||||
item.style.position = "absolute"
|
||||
item.style.left = getComputedStyle(sidenotes).paddingLeft
|
||||
item.style.paddingBottom = item.style.paddingTop = `${BORDER / 2}px`
|
||||
item.style.paddingBottom = item.style.paddingTop = item.style.paddingRight = `${BORDER / 2}px`
|
||||
const itemRect = item.getBoundingClientRect()
|
||||
notes.push({
|
||||
item,
|
||||
@ -427,7 +427,7 @@ if (sidenotes && footnotes) {
|
||||
}
|
||||
// preliminary placement: place in valid regions going down
|
||||
for (const note of notes) {
|
||||
const index = Math.max(inclusions.findLastIndex(inc => (inc.start + note.height) < note.target), 0)
|
||||
const index = Math.max(inclusions.findLastIndex(inc => inc.start < note.target), 0)
|
||||
const next = inclusions.slice(index)
|
||||
.findIndex(inc => (sum(inc.contents.map(x => x.height)) + note.height) < (inc.end - inc.start))
|
||||
inclusions[index + next].contents.push(note)
|
||||
@ -510,6 +510,11 @@ if (sidenotes && footnotes) {
|
||||
})
|
||||
})
|
||||
window.relayout = relayout
|
||||
document.querySelectorAll("img").forEach(x => {
|
||||
x.addEventListener("load", () => {
|
||||
setTimeout(() => relayout(true), 0)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const fixDetailsSummary = () => {
|
||||
|
@ -3,6 +3,9 @@ $content-margin: 1rem
|
||||
$content-width: 40rem
|
||||
$navbar-width: 20rem
|
||||
|
||||
*
|
||||
box-sizing: border-box
|
||||
|
||||
:root
|
||||
--autocol-lightness: 80%
|
||||
--autocol-saturation: 100%
|
||||
@ -167,7 +170,6 @@ button, select, input, textarea, .textarea
|
||||
display: flex
|
||||
img, picture
|
||||
padding-right: 1em
|
||||
height: 8em
|
||||
width: 8em
|
||||
|
||||
.title
|
||||
@ -175,13 +177,15 @@ button, select, input, textarea, .textarea
|
||||
font-weight: 600
|
||||
|
||||
.caption
|
||||
width: calc(100% - 3em)
|
||||
background: lightgray
|
||||
border: 1px solid black
|
||||
padding: 1em
|
||||
margin: 0.5em
|
||||
margin-bottom: 0.5em
|
||||
margin-top: 0.5em
|
||||
img, picture
|
||||
width: 100%
|
||||
p
|
||||
margin: 0
|
||||
|
||||
blockquote
|
||||
padding-left: 0.4rem
|
||||
@ -216,6 +220,7 @@ blockquote
|
||||
.footnotes-list
|
||||
text-align: justify
|
||||
@media (max-width: calc(2 * $content-margin + $content-width + $sidenotes-width))
|
||||
// minwidth 1-pane layout
|
||||
.sidenotes
|
||||
min-width: auto
|
||||
width: auto
|
||||
@ -229,6 +234,7 @@ blockquote
|
||||
display: block
|
||||
|
||||
@media (min-width: calc(2 * $content-margin + $content-width + $sidenotes-width + $navbar-width))
|
||||
// fullwidth 3-pane layout
|
||||
body
|
||||
display: flex
|
||||
.nav-container
|
||||
|
@ -6,11 +6,11 @@ mixin image(src)
|
||||
img(src=src)
|
||||
else
|
||||
picture
|
||||
each val, key in src
|
||||
if key == "_fallback"
|
||||
img(src=val)
|
||||
each pair in src
|
||||
if pair[0] == "_fallback"
|
||||
img(src=pair[1])
|
||||
else
|
||||
source(srcset=val, type=key)
|
||||
source(srcset=pair[1], type=pair[0])
|
||||
|
||||
doctype html
|
||||
html(lang="en")
|
||||
|