diff --git a/assets/images/FTL.jpg b/assets/images/FTL.jpg deleted file mode 100644 index 82f2eeb..0000000 Binary files a/assets/images/FTL.jpg and /dev/null differ diff --git a/assets/images/FTL.png.original b/assets/images/FTL.png.original index 3d0a9c1..c256297 100644 Binary files a/assets/images/FTL.png.original and b/assets/images/FTL.png.original differ diff --git a/assets/images/alphacol.jpg b/assets/images/alphacol.jpg deleted file mode 100644 index 4aec0d2..0000000 Binary files a/assets/images/alphacol.jpg and /dev/null differ diff --git a/assets/images/alphacol.png.original b/assets/images/alphacol.png.original index 612826e..9437f6f 100644 Binary files a/assets/images/alphacol.png.original and b/assets/images/alphacol.png.original differ diff --git a/assets/images/apioform.jpg b/assets/images/apioform.jpg deleted file mode 100644 index 955f180..0000000 Binary files a/assets/images/apioform.jpg and /dev/null differ diff --git a/assets/images/apioform.png.original b/assets/images/apioform.png.original index c41237e..58610fe 100644 Binary files a/assets/images/apioform.png.original and b/assets/images/apioform.png.original differ diff --git a/assets/images/csproblem.jpg b/assets/images/csproblem.jpg deleted file mode 100644 index b6f7388..0000000 Binary files a/assets/images/csproblem.jpg and /dev/null differ diff --git a/assets/images/csproblem.png.original b/assets/images/csproblem.png.original index d5dbf7c..644f034 100644 Binary files a/assets/images/csproblem.png.original and b/assets/images/csproblem.png.original differ diff --git a/assets/images/emu-war.jpg b/assets/images/emu-war.jpg deleted file mode 100644 index 6a9290a..0000000 Binary files a/assets/images/emu-war.jpg and /dev/null differ diff --git a/assets/images/emu-war.png.original b/assets/images/emu-war.png.original index 13128b5..491d7a9 100644 Binary files a/assets/images/emu-war.png.original and b/assets/images/emu-war.png.original differ diff --git a/assets/images/flight.jpg b/assets/images/flight.jpg deleted file mode 100644 index d1e8652..0000000 Binary files a/assets/images/flight.jpg and /dev/null differ diff --git a/assets/images/flight.png.original b/assets/images/flight.png.original index 67eb010..9b6a7c6 100644 Binary files a/assets/images/flight.png.original and b/assets/images/flight.png.original differ diff --git a/assets/images/fractalart.jpg b/assets/images/fractalart.jpg deleted file mode 100644 index d8a0957..0000000 Binary files a/assets/images/fractalart.jpg and /dev/null differ diff --git a/assets/images/fractalart.png.original b/assets/images/fractalart.png.original index 1c052df..f54c882 100644 Binary files a/assets/images/fractalart.png.original and b/assets/images/fractalart.png.original differ diff --git a/assets/images/gol.jpg b/assets/images/gol.jpg deleted file mode 100644 index 0259229..0000000 Binary files a/assets/images/gol.jpg and /dev/null differ diff --git a/assets/images/gol.png.original b/assets/images/gol.png.original index 381fdc9..894c0b1 100644 Binary files a/assets/images/gol.png.original and b/assets/images/gol.png.original differ diff --git a/assets/images/grafana1.png b/assets/images/grafana1.png new file mode 100644 index 0000000..dc87a61 Binary files /dev/null and b/assets/images/grafana1.png differ diff --git a/assets/images/grafana2.png b/assets/images/grafana2.png new file mode 100644 index 0000000..1f84e53 Binary files /dev/null and b/assets/images/grafana2.png differ diff --git a/assets/images/grafana3.png b/assets/images/grafana3.png new file mode 100644 index 0000000..6dcc835 Binary files /dev/null and b/assets/images/grafana3.png differ diff --git a/assets/images/guihacker.jpg b/assets/images/guihacker.jpg deleted file mode 100644 index dccc7f9..0000000 Binary files a/assets/images/guihacker.jpg and /dev/null differ diff --git a/assets/images/guihacker.png.original b/assets/images/guihacker.png.original index d3203af..5e71c22 100644 Binary files a/assets/images/guihacker.png.original and b/assets/images/guihacker.png.original differ diff --git a/assets/images/heavscp.jpg b/assets/images/heavscp.jpg deleted file mode 100644 index 17ae386..0000000 Binary files a/assets/images/heavscp.jpg and /dev/null differ diff --git a/assets/images/heavscp.png.original b/assets/images/heavscp.png.original index df77514..e37a52e 100644 Binary files a/assets/images/heavscp.png.original and b/assets/images/heavscp.png.original differ diff --git a/assets/images/ideas.jpg b/assets/images/ideas.jpg deleted file mode 100644 index 620d5c6..0000000 Binary files a/assets/images/ideas.jpg and /dev/null differ diff --git a/assets/images/ideas.png.original b/assets/images/ideas.png.original index 1826705..42da53d 100644 Binary files a/assets/images/ideas.png.original and b/assets/images/ideas.png.original differ diff --git a/assets/images/incdec.jpg b/assets/images/incdec.jpg deleted file mode 100644 index 321f351..0000000 Binary files a/assets/images/incdec.jpg and /dev/null differ diff --git a/assets/images/incdec.png.original b/assets/images/incdec.png.original index 7fe999b..3ff0eb7 100644 Binary files a/assets/images/incdec.png.original and b/assets/images/incdec.png.original differ diff --git a/assets/images/infipage.jpg b/assets/images/infipage.jpg deleted file mode 100644 index fef2eea..0000000 Binary files a/assets/images/infipage.jpg and /dev/null differ diff --git a/assets/images/infipage.png.original b/assets/images/infipage.png.original index 117cbd7..318b53f 100644 Binary files a/assets/images/infipage.png.original and b/assets/images/infipage.png.original differ diff --git a/assets/images/joe.jpg b/assets/images/joe.jpg deleted file mode 100644 index 2af80d8..0000000 Binary files a/assets/images/joe.jpg and /dev/null differ diff --git a/assets/images/joe.png.original b/assets/images/joe.png.original index 206ca78..ebc3a70 100644 Binary files a/assets/images/joe.png.original and b/assets/images/joe.png.original differ diff --git a/assets/images/lorem.jpg b/assets/images/lorem.jpg deleted file mode 100644 index b9e66c7..0000000 Binary files a/assets/images/lorem.jpg and /dev/null differ diff --git a/assets/images/lorem.png.original b/assets/images/lorem.png.original index f3e6baf..c074eda 100644 Binary files a/assets/images/lorem.png.original and b/assets/images/lorem.png.original differ diff --git a/assets/images/maghammer.jpg b/assets/images/maghammer.jpg deleted file mode 100644 index 62edfa7..0000000 Binary files a/assets/images/maghammer.jpg and /dev/null differ diff --git a/assets/images/maghammer.png.original b/assets/images/maghammer.png.original index 431834f..43c3cbc 100644 Binary files a/assets/images/maghammer.png.original and b/assets/images/maghammer.png.original differ diff --git a/assets/images/minoteaur.jpg b/assets/images/minoteaur.jpg deleted file mode 100644 index 1c3f8c7..0000000 Binary files a/assets/images/minoteaur.jpg and /dev/null differ diff --git a/assets/images/minoteaur.png.original b/assets/images/minoteaur.png.original index 74f270b..ec50f82 100644 Binary files a/assets/images/minoteaur.png.original and b/assets/images/minoteaur.png.original differ diff --git a/assets/images/nemc.jpg b/assets/images/nemc.jpg deleted file mode 100644 index 2f3bb66..0000000 Binary files a/assets/images/nemc.jpg and /dev/null differ diff --git a/assets/images/nemc.png.original b/assets/images/nemc.png.original index 1526da1..7541d27 100644 Binary files a/assets/images/nemc.png.original and b/assets/images/nemc.png.original differ diff --git a/assets/images/new-website.jpg b/assets/images/new-website.jpg deleted file mode 100644 index 19a5eb9..0000000 Binary files a/assets/images/new-website.jpg and /dev/null differ diff --git a/assets/images/new-website.png.original b/assets/images/new-website.png.original index 24f25cf..706240c 100644 Binary files a/assets/images/new-website.png.original and b/assets/images/new-website.png.original differ diff --git a/assets/images/opinion.jpg b/assets/images/opinion.jpg deleted file mode 100644 index 6d33bb5..0000000 Binary files a/assets/images/opinion.jpg and /dev/null differ diff --git a/assets/images/opinion.png.original b/assets/images/opinion.png.original index 812acd5..4335c01 100644 Binary files a/assets/images/opinion.png.original and b/assets/images/opinion.png.original differ diff --git a/assets/images/osbill.jpg b/assets/images/osbill.jpg deleted file mode 100644 index cc01f07..0000000 Binary files a/assets/images/osbill.jpg and /dev/null differ diff --git a/assets/images/osbill.png.original b/assets/images/osbill.png.original index ea538d8..a55eb3d 100644 Binary files a/assets/images/osbill.png.original and b/assets/images/osbill.png.original differ diff --git a/assets/images/osmarkscalculator.jpg b/assets/images/osmarkscalculator.jpg deleted file mode 100644 index 931cb90..0000000 Binary files a/assets/images/osmarkscalculator.jpg and /dev/null differ diff --git a/assets/images/osmarkscalculator.png.original b/assets/images/osmarkscalculator.png.original index 425b1df..06140ca 100644 Binary files a/assets/images/osmarkscalculator.png.original and b/assets/images/osmarkscalculator.png.original differ diff --git a/assets/images/otherstuff.jpg b/assets/images/otherstuff.jpg deleted file mode 100644 index d00bd7d..0000000 Binary files a/assets/images/otherstuff.jpg and /dev/null differ diff --git a/assets/images/otherstuff.png.original b/assets/images/otherstuff.png.original index 71df356..94e1a18 100644 Binary files a/assets/images/otherstuff.png.original and b/assets/images/otherstuff.png.original differ diff --git a/assets/images/phones.jpg b/assets/images/phones.jpg deleted file mode 100644 index fda7e56..0000000 Binary files a/assets/images/phones.jpg and /dev/null differ diff --git a/assets/images/phones.png.original b/assets/images/phones.png.original index 0efe2da..4b82af3 100644 Binary files a/assets/images/phones.png.original and b/assets/images/phones.png.original differ diff --git a/assets/images/points.jpg b/assets/images/points.jpg deleted file mode 100644 index 86620e8..0000000 Binary files a/assets/images/points.jpg and /dev/null differ diff --git a/assets/images/points.png.original b/assets/images/points.png.original index 5d1ea5b..27f89d2 100644 Binary files a/assets/images/points.png.original and b/assets/images/points.png.original differ diff --git a/assets/images/polcal.jpg b/assets/images/polcal.jpg deleted file mode 100644 index 6c8b84d..0000000 Binary files a/assets/images/polcal.jpg and /dev/null differ diff --git a/assets/images/polcal.png.original b/assets/images/polcal.png.original index 868c2b2..a5286db 100644 Binary files a/assets/images/polcal.png.original and b/assets/images/polcal.png.original differ diff --git a/assets/images/progedu.jpg b/assets/images/progedu.jpg deleted file mode 100644 index 38af439..0000000 Binary files a/assets/images/progedu.jpg and /dev/null differ diff --git a/assets/images/progedu.png.original b/assets/images/progedu.png.original index 91ca3d7..01ed31f 100644 Binary files a/assets/images/progedu.png.original and b/assets/images/progedu.png.original differ diff --git a/assets/images/rote.jpg b/assets/images/rote.jpg deleted file mode 100644 index ee1e65d..0000000 Binary files a/assets/images/rote.jpg and /dev/null differ diff --git a/assets/images/rote.png.original b/assets/images/rote.png.original index b2dc4a5..97ff4ac 100644 Binary files a/assets/images/rote.png.original and b/assets/images/rote.png.original differ diff --git a/assets/images/rpncalc2.jpg b/assets/images/rpncalc2.jpg deleted file mode 100644 index f46649d..0000000 Binary files a/assets/images/rpncalc2.jpg and /dev/null differ diff --git a/assets/images/rpncalc2.png.original b/assets/images/rpncalc2.png.original index e1fb4e4..4697c90 100644 Binary files a/assets/images/rpncalc2.png.original and b/assets/images/rpncalc2.png.original differ diff --git a/assets/images/rpncalc3.jpg b/assets/images/rpncalc3.jpg deleted file mode 100644 index c35c6ff..0000000 Binary files a/assets/images/rpncalc3.jpg and /dev/null differ diff --git a/assets/images/rpncalc3.png.original b/assets/images/rpncalc3.png.original index 7270422..fde5d15 100644 Binary files a/assets/images/rpncalc3.png.original and b/assets/images/rpncalc3.png.original differ diff --git a/assets/images/rpncalc4.jpg b/assets/images/rpncalc4.jpg deleted file mode 100644 index ad80664..0000000 Binary files a/assets/images/rpncalc4.jpg and /dev/null differ diff --git a/assets/images/rpncalc4.png.original b/assets/images/rpncalc4.png.original index b0d4ee6..8407acd 100644 Binary files a/assets/images/rpncalc4.png.original and b/assets/images/rpncalc4.png.original differ diff --git a/assets/images/rsapi_diagram_5_alay.png b/assets/images/rsapi_diagram_5_alay.png new file mode 100644 index 0000000..3c5e654 Binary files /dev/null and b/assets/images/rsapi_diagram_5_alay.png differ diff --git a/assets/images/rsapimports.png b/assets/images/rsapimports.png new file mode 100644 index 0000000..90a0cfc Binary files /dev/null and b/assets/images/rsapimports.png differ diff --git a/assets/images/rssgood.jpg b/assets/images/rssgood.jpg deleted file mode 100644 index b21e1cc..0000000 Binary files a/assets/images/rssgood.jpg and /dev/null differ diff --git a/assets/images/rssgood.png.original b/assets/images/rssgood.png.original index 0b31a47..06ff28c 100644 Binary files a/assets/images/rssgood.png.original and b/assets/images/rssgood.png.original differ diff --git a/assets/images/scorer.jpg b/assets/images/scorer.jpg deleted file mode 100644 index 1af2874..0000000 Binary files a/assets/images/scorer.jpg and /dev/null differ diff --git a/assets/images/scorer.png.original b/assets/images/scorer.png.original index 6b5907f..915350b 100644 Binary files a/assets/images/scorer.png.original and b/assets/images/scorer.png.original differ diff --git a/assets/images/srsapi.png.original b/assets/images/srsapi.png.original new file mode 100644 index 0000000..319b9f0 Binary files /dev/null and b/assets/images/srsapi.png.original differ diff --git a/assets/images/stack.jpg b/assets/images/stack.jpg deleted file mode 100644 index 7a82a1e..0000000 Binary files a/assets/images/stack.jpg and /dev/null differ diff --git a/assets/images/stack.png.original b/assets/images/stack.png.original index 7a0f601..7ee93d5 100644 Binary files a/assets/images/stack.png.original and b/assets/images/stack.png.original differ diff --git a/assets/images/themes.jpg b/assets/images/themes.jpg deleted file mode 100644 index 8f8ea70..0000000 Binary files a/assets/images/themes.jpg and /dev/null differ diff --git a/assets/images/themes.png.original b/assets/images/themes.png.original index abbf2fa..c3afdb7 100644 Binary files a/assets/images/themes.png.original and b/assets/images/themes.png.original differ diff --git a/assets/images/tictactoe.jpg b/assets/images/tictactoe.jpg deleted file mode 100644 index 5b894c0..0000000 Binary files a/assets/images/tictactoe.jpg and /dev/null differ diff --git a/assets/images/tictactoe.png.original b/assets/images/tictactoe.png.original index f32ad3b..b2be5ca 100644 Binary files a/assets/images/tictactoe.png.original and b/assets/images/tictactoe.png.original differ diff --git a/assets/images/uptime.png b/assets/images/uptime.png new file mode 100644 index 0000000..5e4ec63 Binary files /dev/null and b/assets/images/uptime.png differ diff --git a/assets/images/whorl.jpg b/assets/images/whorl.jpg deleted file mode 100644 index 8918065..0000000 Binary files a/assets/images/whorl.jpg and /dev/null differ diff --git a/assets/images/whorl.png.original b/assets/images/whorl.png.original index 53ebb30..f8362ad 100644 Binary files a/assets/images/whorl.png.original and b/assets/images/whorl.png.original differ diff --git a/assets/images/wsterm.jpg b/assets/images/wsterm.jpg deleted file mode 100644 index 5169db9..0000000 Binary files a/assets/images/wsterm.jpg and /dev/null differ diff --git a/assets/images/wsterm.png.original b/assets/images/wsterm.png.original index 868c3a1..d7a37aa 100644 Binary files a/assets/images/wsterm.png.original and b/assets/images/wsterm.png.original differ diff --git a/blog/maghammer.md b/blog/maghammer.md index c27cf5d..85301de 100644 --- a/blog/maghammer.md +++ b/blog/maghammer.md @@ -50,29 +50,24 @@ There are also some other datasets handled differently, because the tools I use This is some of what the UI looks like - it is much like a standard Datasette install with a few extra UI elements and some style tweaks I made: -
- -
Viewing browser history through the table view. This is not great on narrower screens. I'm intending to reengineer this a little at some point.
-
-
- -
The redone search-all interface. My plugin makes clickable links pointing to my media server.
-
-
- -
The front page, listing databases and tables and with the search bar.
-
+::: captioned src=/assets/images/maghammer_1.png +Viewing browser history through the table view. This is not great on narrower screens. I'm intending to reengineer this a little at some point. +::: +::: captioned src=/assets/images/maghammer_2.png +The redone search-all interface. My plugin makes clickable links pointing to my media server. +::: +::: captioned src=/assets/images/maghammer_3.png +The front page, listing databases and tables and with the search bar. +::: Being built out of a tool intended for quantitative data processing means that I can, as I mentioned, do some quantitative data processing. While I could in principle do things like count shell/browser history entries by date, this isn't very interesting, and the cooler datasets are logs from my watch (heart rate and step count), although I haven't gotten around to producing nice aggregates from these, and the manually written structured data entries from my journal. For the reasons described earlier I write up a lot of information in journal entries each day, including machine-readable standardized content. I haven't backfilled this for all entries as it requires a lot of work to read through them and write up the tags, but even with only fairly recent entries usable it's still provided significant insight. -
- -
A simple aggregate query of my notes' structured data. Redacted for privacy.
-
-
- -
Not actually a very helpful format.
-
+::: captioned src=/assets/images/maghammer_4.png +A simple aggregate query of my notes' structured data. Redacted for privacy. +::: +::: captioned src=/assets/images/maghammer_5.png +Not actually a very helpful format. +::: While it's not part of the same system, [Meme Search Engine](https://mse.osmarks.net/) is undoubtedly useful to me for rapidly finding images (memetic images) I need or want - so much so that I have a separate internal instance run on my miscellaneous-images-and-screenshots folder. Nobody else seems to even be trying - while there are a lot of demos of CLIP image search engines on GitHub, and I think one with the OpenAI repository, I'm not aware of *production* implementations with the exception of [clip-retrieval](https://github.com/rom1504/clip-retrieval) and the LAION index deployment, and one iPhone app shipping a distilled CLIP. There's not anything like a user-friendly desktop app, which confuses me somewhat, since there's clearly demand amongst people I talked to. Regardless of the reason, this means that Meme Search Engine is quite possibly the world's most advanced meme search tool (since I bothered to design a nice-to-use query UI and online reindexing), although I feel compelled to mention someone's [somewhat horrifying iPhone OCR cluster](https://findthatmeme.com/blog/2023/01/08/image-stacks-and-iphone-racks-building-an-internet-scale-meme-search-engine-Qzrz7V6T.html). Meme Search Engine is not very well-integrated but I usually know which dataset I want to retrieve from anyway. diff --git a/blog/minoteaur.md b/blog/minoteaur.md index fec8b64..1c3ca73 100644 --- a/blog/minoteaur.md +++ b/blog/minoteaur.md @@ -21,10 +21,9 @@ As usually happens with sufficiently large projects which I can't neatly complet It did, at least, get away with supporting its capabilities using impressively little code. While wrong people believe that better software involves more code, I, as an enlightened programmer, recognize that you should write as little code as possible, as more code means more bugs *and* more work to write and maintain it. -
- -
Self-replicating Minoteaur pages on the legacy public Minoteaur. I can no longer actually get it to run, so I don't have any other images.
-
+::: captioned src=/assets/images/minoteaur_1_1.png +Self-replicating Minoteaur pages on the legacy public Minoteaur. I can no longer actually get it to run, so I don't have any other images. +::: After deciding that I really did need something which actually worked even if it wasn't perfect, I settled on... installing [DokuWiki](https://www.dokuwiki.org/dokuwiki) - while a PHP application and not particularly modern featurewise, it was known to be robust, supported *most* of what I wanted, and basically worked. I even dabbled in the horrors of PHP to make some tweaks and plugins I wanted work.[^1] @@ -34,10 +33,9 @@ Prototypes were developed and reengineered for new, exciting Minoteaurs based on Unfortunately, despite the nicer interface and actually-useful Markdown preview mode, these proved frustrating to work on, due to the usual difficulties of maintaining consistency between client code with persistent state and a server which also has persistent state, and I ultimately consigned Minoteaur 2 and 3 to the depths of my project folder in mid-2020. It was also somewhat slow, due to the overhead of parsing Markdown into a parse tree and then rendering that parse tree to virtual DOM and then rendering virtual DOM to actual DOM nodes. -
- -
Minoteaur 2's multitasking UI was ultimately abandoned in favour of just using native browser tabs or windows.
-
+::: captioned src=/assets/images/minoteaur_2.png +Minoteaur 2's multitasking UI was ultimately abandoned in favour of just using native browser tabs or windows. +::: While researching for this I thought that Minoteaur 4 didn't actually exist because of accidents with the numbering scheme, but it turns out that it does, but it's actually Minoteaur 1.5. I had vague memories of a prototype Rust backend for use with a single-page application (with a vaguely RESTful API rather than the usual tightly-coupled RPC designs I use) which I had assumed was developed after the Node.js ones, but it was actually made significantly before those in late 2019. @@ -54,18 +52,15 @@ Nim is sort of how I would design a programming language, both in the sense that It has enough working libraries for things like SQLite and webservers that I thought it worth trying anyway, and it was indeed the most functional Minoteaur at the time, incorporating good SQLite-based search, backlinks, a mostly functional UI, partly style-insensitive links, a reasonably robust parser, a decent UI, and even DokuWiki-like drafts in the editor (a feature I end up using quite often due to things like accidentally closing or refreshing pages). However, I got annoyed again by the server-rendered design, the terrible, terrible code I had to write to directly bind to a C-based GFM library (I think I at least managed to make it not segfault, even though I don't know why), and probably some things I forgot, leading to the *next* version. -
- -
The Minoteaur 6 "two-pane" editor UI.
-
-
- -
Its search mechanism worked, but with some UI problems.
-
-
- -
Minoteaur 6 had an extensive login fallback system.
-
+::: captioned src=/assets/images/minoteaur_6.png +The Minoteaur 6 "two-pane" editor UI. +::: +::: captioned src=/assets/images/minoteaur_6_2.png +Its search mechanism worked, but with some UI problems. +::: +::: captioned src=/assets/images/minoteaur_6_3.png +Minoteaur 6 had an extensive login fallback system. +::: Python is my go-to language for rapid prototyping, i.e. writing poor-quality code very quickly, so it made some sense for me to rewrite in that next in 2021. Minoteaur 7 was a short-lived variant using server rendering, which was rapidly replaced by Minoteaur 7.1, which used a frontend web framework called Svelte for its UI[^3]. @@ -77,14 +72,12 @@ This was actually quite easy to do thanks to the hard work of library developers It also got the furthest yet in terms of general usability, mostly because I implemented file upload, which I think is necessary for any useful notes software (you do, at the very least, want to be able to add and reference images). Ultimately, for some reason I forgot (I think mostly database management this time), I decided that I disliked the code and rewrote it yet again, leading to Minoteaur 8. -
- -
Minoteaur 7.1 introduced a new UI style and more effectively used the width of standard horizontal screens with the search sidebar. It also incorporated page icons for the first time.
-
-
- -
An earlier prototype displaying its search capabilities.
-
+::: captioned src=/assets/images/minoteaur_7.png +Minoteaur 7.1 introduced a new UI style and more effectively used the width of standard horizontal screens with the search sidebar. It also incorporated page icons for the first time. +::: +::: captioned src=/assets/images/minoteaur_7_1.png +An earlier prototype displaying its search capabilities. +::: [Minoteaur 8](https://github.com/osmarks/minoteaur-8) was a rewrite in Rust again, starting in February 2022, using much of the frontend code from Minoteaur 7.1 but with a completely different backend but with a similar architecture, apart from the fact that instead of using SQLite directly and "sensibly", it uses it to store persistent versions of objects (revisions, pageviews or pages) for which live copies and indices are held in memory. Since notes aren't really *that* big, I worked out that even under pessimistic assumptions the RAM requirements would be lower than those of the JS/Python interpreter processes running previous Minoteaurs, which were not particularly large anyway (more on this in my future writing on how all software is terrible), and this made a lot of the code simpler due to not having to limit data structures to what SQLite supports and not having to deal with async IO for read operations. @@ -92,14 +85,12 @@ Since notes aren't really *that* big, I worked out that even under pessimistic a Despite considerable success in making it work to the same extent as previous Minoteaurs (files, search, backlinks, Markdown, etc) and even somewhat further (nicer Markdown syntax, and a three-pane UI), development was mysteriously halted for a while in March and nonmysteriously (some inconsistencies in how context for backlinks versus for search worked which felt annoying to fix) in July after I picked it back up. This April, I happened to look again for some reason, and found that the problem was actually easy if reframed slightly, then did everything else I wanted for usability parity with my DokuWiki install over the course of three days, wrote an import and DokuWiki migration script, redid some of the syntax for more reliable parsing, and finally transitioned away from DokuWiki after slightly less than 4 years. -
- -
Minoteaur's Minoteaur page.
-
-
- -
To allow discovery of interesting content you may have forgotten, Minoteaur incorporates a random pages list.
-
+::: captioned src=/assets/images/minoteaur_8.png +Minoteaur's Minoteaur page. +::: +::: captioned src=/assets/images/minoteaur_8_0.png +To allow discovery of interesting content you may have forgotten, Minoteaur incorporates a random pages list. +::: ## So what does it actually do? diff --git a/blog/ml-workstation.md b/blog/ml-workstation.md index 65dde61..2e5ceeb 100644 --- a/blog/ml-workstation.md +++ b/blog/ml-workstation.md @@ -37,10 +37,9 @@ Intel GPUs have good matrix multiplication accelerators, but their most powerful Many unwary buyers have fallen for the siren song of increasingly cheap used Nvidia Tesla GPUs, since they offer very large VRAM pools at very low cost. However, these are a bad choice unless you *only* need that VRAM. The popular Tesla K80 is 9 years old, with lacking driver support, no FP16, extremely lacking general performance, high power consumption, and no modern optimization efforts, and it's not actually one GPU - it's two on a single card, so you have to deal with parallelizing anything big across GPUs. The next-generation Tesla M40 has similar problems, although it is a single GPU rather than two, and the P40 is not much different, though instead of *no* FP16 it has *unusably slow* FP16[^14]. Even a Tesla P100 is lacking in compute performance compared to newer generations. Datacentre cards newer than that are not available cheaply. There's also some complexity with cooling, since they're designed for server airflow with separate fans, unlike a consumer GPU.[^13] -
- -
It may look innocent, but it is a menace to unaware hobbyists.
-
+::: captioned src=/assets/images/tesla-k80.jpg +It may look innocent, but it is a menace to unaware hobbyists. +::: ### Do not buy workstation cards @@ -72,10 +71,9 @@ GPUs are pretty power-hungry. PCPartPicker will make a good estimate of maximum If you're concerned about reducing your power bill, Ada Lovelace GPUs are generally much more efficient than Ampere due to their newer manufacturing process. You can also power-limit your GPU using `nvidia-smi -pl [power limit in watts]` (note that this must be run each boot in some way): this does reduce performance, but nonlinearly. -
- -
Thanks to "snowy χατγιρλ/acc" on #off-topic for the benchmark. Other GPUs will have different behaviour. This is something of a worst case though - you'll lose less to power limits in real workloads.
-
+::: captioned src=/assets/images/rtx-4090-power-scaling.webp +Thanks to "snowy χατγιρλ/acc" on #off-topic for the benchmark. Other GPUs will have different behaviour. This is something of a worst case though - you'll lose less to power limits in real workloads. +::: ## Other components diff --git a/blog/stack-rsapi.md b/blog/stack-rsapi.md new file mode 100644 index 0000000..4f2a8db --- /dev/null +++ b/blog/stack-rsapi.md @@ -0,0 +1,101 @@ +--- +title: "Site tech stack 2: the unfathomed depths" +description: RSAPI and the rest of my infrastructure. +created: 27/03/2024 +slug: srsapi +--- +The original [Site tech stack](/stack/) article (updated since release somewhat as hardware has improved and software been replaced) covers the basic workings of the public-facing website. However, I run *other* things, some of which are interesting to talk about! I have a number of services for personal use running on the same infrastructure, and several non-web-facing but public services. Here's the latest edition of the handy diagram I made in Graphviz: + +::: captioned src=/assets/images/rsapi_diagram_5_alay.png wide link +This used to be done manually in draw.io, before it became intractable to run the layouts by hand. You may want to open it in fullscreen. +::: + +This is split into several boxes indicating the various servers several subsystems run on. As I mention in the comments of the old article, I have a physical server running the actual compute tasks (`protagonism`), but a VPS (`procyon`) is where your HTTP requests are initially going. Since it tends to have better uptime[^1], it also runs the [uptime monitoring system](https://status.osmarks.net/) onstat3, my [Discord bot](https://github.com/osmarks/autobotrobot), and a few other directly network-facing things: osmarksDNS[^3], an IRC server ([APIONET](https://apionet.gh0.pw/)) and smtp2rss. + +If you know what SMTP stands for, hearing "smtp2rss" may have confused you (if you have not talked to me much) or worried you (if you have). Don't worry: it's perfectly sane and reasonable. I [like RSS](/rssgood/), but many people try to email me things, without an RSS fallback[^2]. I dislike minor workflow inconveniences and am willing to throw arbitrary amounts of technology and engineering at them (sometimes), so I wrote a [Python script](https://github.com/osmarks/random-stuff/blob/master/smtp2rss.py) to take inbound emails on a spare domain and expose them as RSS feeds. As a handy bonus, it can provide disposable mailboxes for signing up to services. + +A fun quirk of the nginx installation on `procyon` is that, since I wanted it to not be able to decrypt requests to `protagonism` (I don't entirely trust it, and duplicating the certificate issuance programs on each would be irritating), I use [ngx_stream_ssl_preread](https://nginx.org/en/docs/stream/ngx_stream_ssl_preread_module.html) to forward still-encrypted TLS connections either to itself (on another port) or `protagonism`'s reverse proxy. As janky as this sounds, it does seem to work fine, except for one extremely-hard-to-reproduce bug I suspect might be related where users sometimes get shown 404 pages or the status page incorrectly. Traffic is routed over [Tailscale](https://tailscale.com/) using [Headscale](https://github.com/juanfont/headscale)[^5]. + +Several of `protagonism`'s services are mostly-self-contained personal-use applications, such as [Minoteaur](/minoteaur/) (notes), [ankisyncd](https://github.com/ankicommunity/ankicommunity-sync-server/) (flashcards), [atuin](https://github.com/atuinsh/atuin) (shell history) and [calibre-web](https://github.com/janeczku/calibre-web) (books). The rest are somewhat more interesting, in that they do more and are in some cases publicly accessible. For example, [SPUDNET](https://d.gh0.pw/doku.php?id=gtech:spudnet). It was built to serve the needs of an ["operating system"](https://potatos.madefor.cc/) for ComputerCraft (a Minecraft computer mod) by providing remote debugging services. Originally built about six years ago, it somehow still works with relatively minor changes (new protocol support). It provides bidirectional many-to-one and many-to-many communications over websocket, with an unnecessarily sophisticated authentication system, as well as HTTP long polling fallbacks and incident reports. [Skynet](https://github.com/osmarks/skynet) is a somewhat simpler version. + +I also have a monitoring system using [VictoriaMetrics](https://docs.victoriametrics.com/)[^4] and [Grafana](https://grafana.com/). VictoriaMetrics periodically scrapes services for metrics and stores time series, and Grafana can plot them. This is a fairly standard setup, and lots of software exposes Prometheus-compatible metrics itself or has an exporter available (e.g. the [node_exporter](https://github.com/prometheus/node_exporter) for general Linux machine status and the [PostgreSQL exporter](https://github.com/prometheus-community/postgres_exporter)). I went slightly further by exposing metrics in most of my custom applications, so I have, for instance, nice dashboards from my Discord bot. These used to be public, but apparently that exposing any dashboard in Grafana allows users to read any data out of the backend, which was a bit of a security issue. One might reasonably question how much use I get out of these, as I don't get enough traffic to have to debug performance issues much, but they do look nice and their presence is calming. + +::: captioned src=/assets/images/grafana1.png +My customized node-exporter dashboard. I still need more storage. +::: +::: captioned src=/assets/images/grafana2.png +AutoBotRobot monitoring. +::: + +The most important component I have is undoubtedly RSAPI, the highly custom integration script which does about twenty different things since putting them in different services would have been annoying. Many things I need to do share the same basic building blocks - a database or simple state storage, timers and an HTTP server - and while it would be *possible* though tricky to factor this out into a library and write several microservices, the deployment would be harder to manage with my tools and many of the parts have to interoperate anyway. Some would consider a 1600-line monolithic Python program plugged into 10 different APIs "bad", but, freed from Conway's law[^6], I think it is actually the most efficient way to do this. For whatever reason I can track its structure mentally very efficiently, so it's not hard to work on. + +::: captioned src=/assets/images/rsapimports.png +The RSAPI imports section. +::: + +RSAPI has a wide range of functions, having grown from a short Flask application which served [fortunes](https://wiki.archlinux.org/title/Fortune) to [PotatOS](https://potatos.madefor.cc/) by accretion of additional capabilities as they were needed. The exact history has been lost to the halcyon days of poor version control and backups, but it was built in roughly this order: + +* Initial version built: served fortunes and [https://schlockmercenary.fandom.com/wiki/The_Seventy_Maxims_of_Maximally_Effective_Mercenaries](https://schlockmercenary.fandom.com/wiki/The_Seventy_Maxims_of_Maximally_Effective_Mercenaries) over HTTP. +* "Currently playing" support for my internet radio server, via integration with MPD. +* youtube-dl web frontend and very basic login. +* IRC bot for server status and MPD status. +* Rewrite from Flask/gevent to asyncio/aiohttp. +* DNS to comments (and IRC) bridge - converts specially formatted DNS queries to a subdomain to comments on a certain page. +* "Currently playing" support expanded with listener counts. +* [Miniflux](https://miniflux.app/) (RSS reader) to Discord bridge, and (unnecessary, since it has an RSS button I somehow ignored) RoyalRoad fiction to RSS bridge. +* Random video selector (from local media folders). +* [IncDec](https://osmarks.net/incdec/)/IRC Bridge. +* Cancelled rewrite to make it more modular. Migration of internal databases from SQLite to LevelDB. +* Live chat on internet radio (specialized bridge to IRC). +* NASA [Astronomy Picture of the Day](https://apod.nasa.gov/apod/) fetcher implemented. Daily task scheduler built. +* Wordnik Word of the Day fetcher implemented. +* ComputerCraft to Prometheus metrics bridge. +* "[Webmaze](https://r.osmarks.net/maze/ZSwLxemYUq59J-Hcr-rx0ejcJmMvrjAhA4Nxa7KcBgiNmmTVa8ZxDHVw-ZVZXhFMxj_6kA)" prototype - an infinite partly connected 3D grid implemented fully statelessly, intended for a hypothetical adventure game which never materialized. +* [Freefall](http://freefall.purrsia.com/) (webcomic) fetcher implemented. +* Migration from youtube-dl to yt_dlp. +* Cross-device scratchpad/clipboard for copying short text between computers. +* For some reason, a script which infinitely generates primes, digits of e and digits of pi, reads them out using bad TTS, and sends them as an internet radio stream[^9]. +* Random bytes API. +* Basic service status page for a tablet (to be glued to walls). +* Migration back to SQLite. Proper database migration system. +* URL shortener - in conjunction with my [really short domain](https://0t.lt/), it can produce really short URLs, as well as ZWS-based URLs and two-word URLs like [https://0t.lt/YearningPried](https://0t.lt/YearningPried) - these use a special prefix-free wordlist so they can also be typed in uncapitalized unambiguously. +* [zzcxz](https://zzcxz.citrons.xyz/) telephony interface. +* Tablet status interface enhanced with list of failed services (from systemd). +* Personal event logger. +* Remote to local calendar synchronizer. +* Weather and calendar updates (for the tablet). +* [Threat Update (Twitter)](https://twitter.com/threat_update) scraper. +* [SwitchCraft](https://sc3.io/) player surveillance (Dynmap). +* RSS feed for random memes (for XScreensaver). +* Better login system - multiple users, SSO for other services via [Nginx authentication subrequests](https://docs.nginx.com/nginx/admin-guide/security-controls/configuring-subrequest-authentication/), basicauth option for non-interactive systems. +* Key/value storage backend for PotatOS, due to the shutdown of the random free API it used before. +* Internal LLM-based [Threat Updates](https://r.osmarks.net/threat-update) system[^8], to replace the archive of historical ones and Twitter scraper. I was too lazy to work out how to draw nicely line-wrapped text in images in Python, so this actually invokes a ComputerCraft emulator, runs the Threat Update implementation on that, dumps its virtual screen, and renders that to an image. +* The new comments system, replacing [Isso](https://github.com/isso-comments/isso). It supports ominous AI faces (from StyleGAN2, thanks to [StyleGANCpp](https://github.com/podgorskiy/StyleGANCpp/)[^7]), leftvotes/rightvotes for greater user expression, SSO integration, better threading, and lower client resource use. + +I'm especially proud of the ComputerCraft to Prometheus metrics bridge. While it's only about 20 lines of code (plus the ComputerCraft side), it does allow me to feel cool about being able to monitor meaninglessly large numbers in great detail. There are similar Factorio mods, which I'll probably use next time I play. + +::: captioned src=/assets/images/grafana3.png +My base's reactor powering up on a recent tech modpack server, as visualized from Grafana. +::: + +I also have some custom inference servers backing [Meme Search Engine](https://mse.osmarks.net/) and [Maghammer](/maghammer/), in addition to an ExllamaV2-based LLM API used in PotatOS. Early prototypes loaded the models in-process, but this was very inflexible: restarts were slow, only one process at a time could use them, and it effectively required that consuming code be written in Python. The servers are basic (no automatic batching and few optimizations), but are presently good enough to handle traffic. The CLIP one is in fact open as part of [Meme Search Engine](https://github.com/osmarks/meme-search-engine/blob/master/clip_server.py). + +I haven't covered *every* osmarks.net service in this post, or even all the ones in the slightly outdated diagram above, but I think I got the most interesting ones. I hope this was informative, and did not accidentally make people notice horrible security issues I missed. + +[^1]: Much better uptime: ![An SSH session on procyon saying "up 588 days, 38 min".](/assets/images/uptime.png) + +[^2]: Ironically, they are doing this via mailing list services which absolutely could also offer RSS if they wanted to (Mailchimp does, even, as an option). They probably don't want to for "engagement" reasons. + +[^3]: osmarksDNS is less interesting, and refers to a DNS over HTTPS server and recursive resolver installed locally, I think because of an issue with bootstrapping dnscrypt-proxy I had years ago. That was fixed another way, but I never had a compelling reason to shut it down. + +[^4]: This used to be Prometheus, but I swapped VictoriaMetrics in to reduce storage requirements. + +[^5]: I like Tailscale's ease of use, but it's horrifyingly CPU-intensive for no obvious reason, and `procyon` is not very powerful. This would be a problem if I had traffic. + +[^6]: "The structure of any system designed by an organization is isomorphic to the structure of the organization." You could argue that this is more "directly in line with Conway's law" than "freed from it", but ignore that. + +[^7]: It seems to have a gender bias problem, presumably due to dataset or potentially broken implementation. I wanted to use Stable Diffusion, but the compute costs are too bad to run it on CPU and I don't have the free VRAM to load it on GPU constantly. + +[^8]: Here's the latest: + +[^9]: That one bothers me. It contains an algorithm for streaming digits of π which I clearly got from GitHub somewhere, but the one for e is written in my style but isn't something I understand or recognize. \ No newline at end of file diff --git a/blog/stack.md b/blog/stack.md index aebe02f..19ed011 100644 --- a/blog/stack.md +++ b/blog/stack.md @@ -12,12 +12,12 @@ The main site itself, which you're currently reading, is in fact just a simple s Being static files, many, many different webservers could have been used for this site. In practice, it's mostly alternated randomly between [caddy](https://caddyserver.com/) (a more recent, Go-based webserver with automatic LetsEncrypt integration) and [nginx](https://nginx.org/) (an older and more powerful but slightly quirky program) - caddy generally had easier configuration, but I arbitrarily preferred nginx in some ways. After caddy v2 suddenly required me to rewrite my configuration and introduced a bunch of weird issues, I permanently switched over to nginx and haven't changed back. The configuration file is now 600 lines or so, even with inclusion of includes to shorten things, but it... works, at least. This is mostly to accommodate the bizzarely large set of subdomains I now have for various people, and reverse proxy configuration for backend services. I also use a custom-compiled build of nginx with HTTP/3 (QUIC) support and some modules compiled in. -Some of these backend things are only for personal use, but a few are related to the site itself. For example, the comment server is a standalone Python program, [isso](https://posativ.org/isso/), with corresponding JS embedded in each page. This works pretty well, but has lead to some weird quirkiness, such as each separate 404-erroring URL having its own list of comments. There's also the Random Stuff API, a custom assemblage of about 15 different Python libraries and external programs which, while technically not linked on the site, does interact with other projects like [PotatOS](https://git.osmarks.net/osmarks/potatOS/), and internal services on the same infrastructure like my [RSS reader](https://miniflux.app/). The images subdomain also uses a [PHP program](https://larsjung.de/h5ai/) to generate a nice searchable index; in fact, it is one of two the only PHP things I have unfortunately not yet been able to purge[^1]. There also used to be a publicly available status page using some custom code, but this doesn't work very well and has now been dropped; previously I had a Grafana (and earlier Netdata) instance there, but this has now been cancelled because it leaks a worrying amount of information. +Some of these backend things are only for personal use, but a few are related to the site itself. For example, the comment server is ~~a standalone Python program, [isso](https://posativ.org/isso/),~~ now part of [RSAPI](/srsapi/) with corresponding JS embedded in each page. This works pretty well, but has lead to some weird quirkiness, such as each separate 404-erroring URL having its own list of comments. There's also the Random Stuff API, a custom assemblage of about 15 different Python libraries and external programs which, while technically not linked on the site, does interact with other projects like [PotatOS](https://git.osmarks.net/osmarks/potatOS/), and internal services on the same infrastructure like my [RSS reader](https://miniflux.app/). The images subdomain also uses a [PHP program](https://larsjung.de/h5ai/) to generate a nice searchable index; in fact, it is one of two the only PHP things I have unfortunately not yet been able to purge[^1]. There also used to be a publicly available status page using some custom code, but this doesn't work very well and has now been dropped; previously I had a Grafana (and earlier Netdata) instance there, but this has now been cancelled because it leaks a worrying amount of information. As for the underlying OS everything runs on, I currently use [Arch Linux](https://i.osmarks.net/memes-or-something/arch-btw.png) (as well as Alpine on a few lower-resourced cloud servers). Some form of Linux is inevitable - BSDs aren't really compatible with much, and Windows is obviously unsuited for server duty - but I mostly use Arch for its stability (this sounds sarcastic, but I've actually found it to be very reliable with regular updates), wide range of packages (particularly from the AUR; as I don't really run critical production infrastructure, I can generally afford to compile stuff from source a lot), and better general ease-of-use than Alpine. As much as I vaguely resent it, this is mostly down to systemd - despite it being a horrific bloated monolith, `journalctl` is very convenient and unit files are pleasant and easy to write compared to the weird OpenRC scripts Alpine uses. I am actually considering yet another redesign, however; switching to a dynamic site implementation instead would allow me to integrate the comment system and achievement system better, make things like the "from other blogs" tiles actually update at reasonable intervals, and arbitrarily A/B test users, although it would break some nice things like this site's very aggressive caching and fast serving. Please leave your thoughts or lack of thoughts on this in the comments. -[^1]: The previous one was DokuWiki, now replaced with Minoteaur. +[^1]: The previous one was DokuWiki, now replaced with [Minoteaur](/minoteaur/). [^2]: My next upgrade is probably going to be more SSD space, since I'm *somehow* running out of that. \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 5b7ed4f..2b5c783 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,8 +20,9 @@ "handlebars": "^4.7.6", "html-minifier": "^4.0.0", "idb": "^7.1.1", - "markdown-it": "^13.0.1", + "markdown-it": "^14.1.0", "markdown-it-anchor": "^8.6.7", + "markdown-it-container": "^4.0.0", "markdown-it-footnote": "^3.0.3", "mustache": "^4.0.1", "nanoid": "^2.1.11", @@ -777,9 +778,9 @@ } }, "node_modules/entities": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz", - "integrity": "sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", "engines": { "node": ">=0.12" }, @@ -1237,11 +1238,11 @@ } }, "node_modules/linkify-it": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-4.0.1.tgz", - "integrity": "sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", "dependencies": { - "uc.micro": "^1.0.1" + "uc.micro": "^2.0.0" } }, "node_modules/lower-case": { @@ -1261,18 +1262,19 @@ } }, "node_modules/markdown-it": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-13.0.1.tgz", - "integrity": "sha512-lTlxriVoy2criHP0JKRhO2VDG9c2ypWCsT237eDiLqi09rmbKoUetyGHq2uOIRoRS//kfoJckS0eUzzkDR+k2Q==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", "dependencies": { "argparse": "^2.0.1", - "entities": "~3.0.1", - "linkify-it": "^4.0.1", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" }, "bin": { - "markdown-it": "bin/markdown-it.js" + "markdown-it": "bin/markdown-it.mjs" } }, "node_modules/markdown-it-anchor": { @@ -1284,6 +1286,11 @@ "markdown-it": "*" } }, + "node_modules/markdown-it-container": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/markdown-it-container/-/markdown-it-container-4.0.0.tgz", + "integrity": "sha512-HaNccxUH0l7BNGYbFbjmGpf5aLHAMTinqRZQAEQbMr2cdD3z91Q6kIo1oUn1CQndkT03jat6ckrdRYuwwqLlQw==" + }, "node_modules/markdown-it-footnote": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz", @@ -1295,9 +1302,9 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, "node_modules/mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" }, "node_modules/mime-db": { "version": "1.52.0", @@ -1591,6 +1598,14 @@ "once": "^1.3.1" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "engines": { + "node": ">=6" + } + }, "node_modules/ramda": { "version": "0.26.1", "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz", @@ -1901,9 +1916,9 @@ } }, "node_modules/uc.micro": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", - "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" }, "node_modules/uglify-js": { "version": "3.13.10", @@ -2429,9 +2444,9 @@ } }, "entities": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-3.0.1.tgz", - "integrity": "sha512-WiyBqoomrwMdFG1e0kqvASYfnlb0lp8M5o5Fw2OFq1hNZxxcNk8Ik0Xm7LxzBhuidnZB/UtBqVCgUz3kBOP51Q==" + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" }, "esbuild": { "version": "0.19.6", @@ -2751,11 +2766,11 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" }, "linkify-it": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-4.0.1.tgz", - "integrity": "sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", + "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", "requires": { - "uc.micro": "^1.0.1" + "uc.micro": "^2.0.0" } }, "lower-case": { @@ -2772,15 +2787,16 @@ } }, "markdown-it": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-13.0.1.tgz", - "integrity": "sha512-lTlxriVoy2criHP0JKRhO2VDG9c2ypWCsT237eDiLqi09rmbKoUetyGHq2uOIRoRS//kfoJckS0eUzzkDR+k2Q==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", + "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", "requires": { "argparse": "^2.0.1", - "entities": "~3.0.1", - "linkify-it": "^4.0.1", - "mdurl": "^1.0.1", - "uc.micro": "^1.0.5" + "entities": "^4.4.0", + "linkify-it": "^5.0.0", + "mdurl": "^2.0.0", + "punycode.js": "^2.3.1", + "uc.micro": "^2.1.0" }, "dependencies": { "argparse": { @@ -2796,15 +2812,20 @@ "integrity": "sha512-FlCHFwNnutLgVTflOYHPW2pPcl2AACqVzExlkGQNsi4CJgqOHN7YTgDd4LuhgN1BFO3TS0vLAruV1Td6dwWPJA==", "requires": {} }, + "markdown-it-container": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/markdown-it-container/-/markdown-it-container-4.0.0.tgz", + "integrity": "sha512-HaNccxUH0l7BNGYbFbjmGpf5aLHAMTinqRZQAEQbMr2cdD3z91Q6kIo1oUn1CQndkT03jat6ckrdRYuwwqLlQw==" + }, "markdown-it-footnote": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/markdown-it-footnote/-/markdown-it-footnote-3.0.3.tgz", "integrity": "sha512-YZMSuCGVZAjzKMn+xqIco9d1cLGxbELHZ9do/TSYVzraooV8ypsppKNmUJ0fVH5ljkCInQAtFpm8Rb3eXSrt5w==" }, "mdurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", - "integrity": "sha1-/oWy7HWlkDfyrf7BAP1sYBdhFS4=" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" }, "mime-db": { "version": "1.52.0", @@ -3059,6 +3080,11 @@ "once": "^1.3.1" } }, + "punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==" + }, "ramda": { "version": "0.26.1", "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.26.1.tgz", @@ -3264,9 +3290,9 @@ } }, "uc.micro": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-1.0.6.tgz", - "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" }, "uglify-js": { "version": "3.13.10", diff --git a/package.json b/package.json index c4db6a1..d5978b7 100644 --- a/package.json +++ b/package.json @@ -15,8 +15,9 @@ "handlebars": "^4.7.6", "html-minifier": "^4.0.0", "idb": "^7.1.1", - "markdown-it": "^13.0.1", + "markdown-it": "^14.1.0", "markdown-it-anchor": "^8.6.7", + "markdown-it-container": "^4.0.0", "markdown-it-footnote": "^3.0.3", "mustache": "^4.0.1", "nanoid": "^2.1.11", diff --git a/src/avif_compact.sh b/src/avif_compact.sh new file mode 100755 index 0000000..5f36af1 --- /dev/null +++ b/src/avif_compact.sh @@ -0,0 +1,5 @@ +#!/bin/sh +file=$(mktemp /tmp/tmp.XXXXXXXXXX.png) +convert "$1" -resize 25% "$file" +avifenc -s 0 -q 50 "$file" $2 +rm "$file" \ No newline at end of file diff --git a/src/global.json b/src/global.json index a834505..4344ca5 100644 --- a/src/global.json +++ b/src/global.json @@ -30,7 +30,7 @@ "https://www.rtl-sdr.com/feed/", "https://astralcodexten.substack.com/feed", "https://www.rifters.com/crawl/?feed=rss2", - "https://drewdevault.com/feed.xml", + "https://drewdevault.com/blog/index.xml", "https://qntm.org/rss.php", "https://aphyr.com/posts.atom", "https://os.phil-opp.com/rss.xml" diff --git a/src/index.js b/src/index.js index 047d8c5..6bec8ef 100644 --- a/src/index.js +++ b/src/index.js @@ -76,9 +76,42 @@ globalData.hashBG = hashBG const removeExtension = x => x.replace(/\.[^/.]+$/, "") +const mdutils = MarkdownIt().utils +const renderContainer = (tokens, idx) => { + let opening = true + if (tokens[idx].type === "container__close") { + let nesting = 0 + for (; tokens[idx].type !== "container__open" && nesting !== 1; idx--) { + nesting += tokens[idx].nesting + } + opening = false + } + const m = tokens[idx].info.trim().split(" "); + const blockType = m[0] + + const options = {} + for (const arg of m.slice(1)) { + const [k, v] = arg.split("=", 2) + options[k] = v ?? true + } + + if (opening) { + if (blockType === "captioned") { + const link = `` + return `
${options.link ? link : ""}${options.link ? "" : ""}` + } + } else { + if (blockType === "captioned") { + return `
` + } + } + throw new Error(`unrecognized blockType ${blockType}`) +} + const readFile = path => fsp.readFile(path, { encoding: "utf8" }) const anchor = require("markdown-it-anchor") const md = new MarkdownIt({ html: true }) + .use(require("markdown-it-container"), "", { render: renderContainer, validate: params => true }) .use(require("markdown-it-footnote")) .use(anchor, { permalink: anchor.permalink["headerLink"]({ @@ -332,10 +365,9 @@ const doImages = async () => { (await fse.readdir(path.join(assetsDir, "images"), { encoding: "utf-8" })).map(async image => { if (image.endsWith(".original")) { // generate alternative formats const stripped = image.replace(/\.original$/).split(".").slice(0, -1).join(".") - globalData.images[stripped] = {} const fullPath = path.join(assetsDir, "images", image) const stat = await fse.stat(fullPath) - const writeFormat = async (name, ext, mime, cmd, supplementaryArgs) => { + const writeFormat = async (name, ext, cmd, supplementaryArgs, suffix="") => { let bytes = readCache(`images/${stripped}/${name}`, null, stat.mtimeMs) const destFilename = stripped + ext const destPath = path.join(outAssets, "images", destFilename) @@ -350,10 +382,15 @@ const doImages = async () => { await fsp.writeFile(destPath, bytes) } - globalData.images[stripped][mime] = "/assets/images/" + destFilename + return "/assets/images/" + destFilename } - await writeFormat("avif", ".avif", "image/avif", "avifenc", ["-s", "0", "-q", "20"]) - await writeFormat("jpeg-scaled", ".jpg", "_fallback", "convert", ["-resize", "25%", "-format", "jpeg"]) + const avif = await writeFormat("avif", ".avif", "avifenc", ["-s", "0", "-q", "20"], " 2x") + const avifc = await writeFormat("avif-compact", ".c.avif", path.join(srcDir, "avif_compact.sh"), []) + const jpeg = await writeFormat("jpeg-scaled", ".jpg", "_fallback", "convert", ["-resize", "25%", "-format", "jpeg"]) + globalData.images[stripped] = [ + ["image/avif", `${avifc}, ${avif} 2x`], + ["_fallback", jpeg] + ] } else { globalData.images[image.split(".").slice(0, -1).join(".")] = "/assets/images/" + image } diff --git a/src/page.js b/src/page.js index e3a42a5..82da967 100644 --- a/src/page.js +++ b/src/page.js @@ -417,7 +417,7 @@ if (sidenotes && footnotes) { const linkRect = link.getBoundingClientRect() item.style.position = "absolute" item.style.left = getComputedStyle(sidenotes).paddingLeft - item.style.paddingBottom = item.style.paddingTop = `${BORDER / 2}px` + item.style.paddingBottom = item.style.paddingTop = item.style.paddingRight = `${BORDER / 2}px` const itemRect = item.getBoundingClientRect() notes.push({ item, @@ -427,7 +427,7 @@ if (sidenotes && footnotes) { } // preliminary placement: place in valid regions going down for (const note of notes) { - const index = Math.max(inclusions.findLastIndex(inc => (inc.start + note.height) < note.target), 0) + const index = Math.max(inclusions.findLastIndex(inc => inc.start < note.target), 0) const next = inclusions.slice(index) .findIndex(inc => (sum(inc.contents.map(x => x.height)) + note.height) < (inc.end - inc.start)) inclusions[index + next].contents.push(note) @@ -510,6 +510,11 @@ if (sidenotes && footnotes) { }) }) window.relayout = relayout + document.querySelectorAll("img").forEach(x => { + x.addEventListener("load", () => { + setTimeout(() => relayout(true), 0) + }) + }) } const fixDetailsSummary = () => { diff --git a/src/style.sass b/src/style.sass index 0bea5d9..81852d7 100644 --- a/src/style.sass +++ b/src/style.sass @@ -3,6 +3,9 @@ $content-margin: 1rem $content-width: 40rem $navbar-width: 20rem +* + box-sizing: border-box + :root --autocol-lightness: 80% --autocol-saturation: 100% @@ -167,7 +170,6 @@ button, select, input, textarea, .textarea display: flex img, picture padding-right: 1em - height: 8em width: 8em .title @@ -175,13 +177,15 @@ button, select, input, textarea, .textarea font-weight: 600 .caption - width: calc(100% - 3em) background: lightgray border: 1px solid black padding: 1em - margin: 0.5em + margin-bottom: 0.5em + margin-top: 0.5em img, picture width: 100% + p + margin: 0 blockquote padding-left: 0.4rem @@ -216,6 +220,7 @@ blockquote .footnotes-list text-align: justify @media (max-width: calc(2 * $content-margin + $content-width + $sidenotes-width)) + // minwidth 1-pane layout .sidenotes min-width: auto width: auto @@ -229,6 +234,7 @@ blockquote display: block @media (min-width: calc(2 * $content-margin + $content-width + $sidenotes-width + $navbar-width)) + // fullwidth 3-pane layout body display: flex .nav-container diff --git a/templates/layout.pug b/templates/layout.pug index 27158ae..4c23f88 100644 --- a/templates/layout.pug +++ b/templates/layout.pug @@ -6,11 +6,11 @@ mixin image(src) img(src=src) else picture - each val, key in src - if key == "_fallback" - img(src=val) + each pair in src + if pair[0] == "_fallback" + img(src=pair[1]) else - source(srcset=val, type=key) + source(srcset=pair[1], type=pair[0]) doctype html html(lang="en")