Rebuild "other blogs" service without Openring

src/global.json

@@ -28,16 +28,25 @@
         "Paperclip rights are human rights.",
         "Humans aren't even AGI."
     ],
-    "feeds": [
-        "https://www.science.org/blogs/pipeline/feed",
-        "https://www.rtl-sdr.com/feed/",
-        "https://astralcodexten.substack.com/feed",
-        "https://www.rifters.com/crawl/?feed=rss2",
-        "https://drewdevault.com/blog/index.xml",
-        "https://qntm.org/rss.php",
-        "https://aphyr.com/posts.atom",
-        "https://os.phil-opp.com/rss.xml"
-    ],
+    "feeds": {
+        "In the Pipeline": "https://www.science.org/blogs/pipeline/feed",
+        "rtl-sdr.com": "https://www.rtl-sdr.com/feed/",
+        "Astral Codex Ten": "https://astralcodexten.substack.com/feed",
+        "Peter Watts": "https://www.rifters.com/crawl/?feed=rss2",
+        "Drew DeVault": "https://drewdevault.com/blog/index.xml",
+        "Things Of Interest":"https://qntm.org/rss.php",
+        "Chips and Cheese": "https://chipsandcheese.com/feed",
+        "BOFH": "https://www.theregister.com/offbeat/bofh/headlines.atom",
+        "Don't Worry About the Vase": "https://thezvi.substack.com/feed",
+        "KGOnTech": "https://kguttag.com/feed/",
+        "Overcoming Bias": "https://www.overcomingbias.com/feed",
+        "Construction Physics": "https://www.construction-physics.com/feed",
+        "Factorio": "https://www.factorio.com/blog/rss",
+        "The Eldraeverse": "https://eldraeverse.com/feed/",
+        "STH Newsletter Archive Feed": "https://us13.campaign-archive.com/feed?u=45c4ca50425f376ea593dcd22&id=c344a80940",
+        "Graphcore Research": "https://graphcore-research.github.io/feed.xml",
+        "ToughSF": "https://toughsf.blogspot.com/rss.xml"
+    },
     "dateFormat": "YYYY-MM-DD",
     "microblogSource": "https://b.osmarks.net/outbox",
     "buttons": [

src/index.js

@@ -26,6 +26,7 @@ const htmlparser2 = require("htmlparser2")
 const cssSelect = require("css-select")
 const domSerializer = require("dom-serializer")
 const domutils = require("domutils")
+const feedExtractor = require("@extractus/feed-extractor")
 
 const fts = require("./fts.mjs")
 
@@ -149,7 +150,7 @@ const renderContainer = (tokens, idx) => {
 
 const readFile = path => fsp.readFile(path, { encoding: "utf8" })
 const anchor = require("markdown-it-anchor")
-const { htmlToText } = require("html-to-text")
+
 const md = new MarkdownIt({ html: true })
     .use(require("markdown-it-container"), "", { render: renderContainer, validate: params => true })
     .use(require("markdown-it-footnote"))
@@ -355,10 +356,14 @@ const writeCache = (k, v, ts=Date.now()) => {
 }
 
 const DESC_CUT_LEN = 256
+
+const cutDesc = desc => desc.length > DESC_CUT_LEN ? `${desc.slice(0, DESC_CUT_LEN)}...` : desc
+
 const fetchMicroblog = async () => {
     const cached = readCache("microblog", 60*60*1000)
     if (cached) {
         globalData.microblog = cached
+        console.log(chalk.yellow("Using cached microblog"))
     } else {
         // We have a server patch which removes the 20-post hardcoded limit.
         // For some exciting reason microblog.pub does not expose pagination in the *API* components.
@@ -375,7 +380,7 @@ const fetchMicroblog = async () => {
             url: post.object.id,
             timestamp: dayjs(post.object.published),
             html: post.object.content,
-            description: desc.length > DESC_CUT_LEN ? desc.slice(0, DESC_CUT_LEN) + "..." : desc,
+            description: cutDesc(desc),
             ignoreDescription: true
         })
     }
@@ -389,16 +394,53 @@ const fetchMicroblog = async () => {
     })))
 }
 
-const runOpenring = async () => {
-    const cached = readCache("openring", 60*60*1000)
-    if (cached) { globalData.openring = cached; return }
-    // wildly unsafe but only runs on input from me anyway
-    const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < ./src/openring.html`
-    console.log(chalk.keyword("orange")("Openring:") + " " + arg)
-    const out = await util.promisify(childProcess.exec)(arg)
-    console.log(chalk.keyword("orange")("Openring:") + "\n" + out.stderr.trim())
-    globalData.openring = minifyHTML(out.stdout)
-    writeCache("openring", globalData.openring)
+const fetchFeeds = async () => {
+    const cached = readCache("feeds", 60*60*1000)
+    if (cached) {
+        globalData.openring = cached
+        console.log(chalk.yellow("Using cached feeds"))
+    } else {
+        globalData.openring = {}
+        const getOneFeed = async url => {
+            try {
+                const data = await axios.get(url, { headers: { "User-Agent": "osmarks.net static site compiler" } })
+                return feedExtractor.extractFromXml(data.data)
+            } catch (e) {
+                console.warn(`${chalk.red("Failed to fetch")} ${url}: ${e.message} ${e.errors && e.errors.map(x => x.message).join("\n")}`)
+            }
+        }
+        await Promise.all(Object.entries(globalData.feeds).map(async ([name, url]) => {
+            const feed = await getOneFeed(url)
+            if (feed) {
+                globalData.openring[name] = feed
+            }
+        }))
+        writeCache("feeds", globalData.openring)
+    }
+    
+    const entries = []
+    for (const [name, feed] of Object.entries(globalData.openring)) {
+        for (const entry of feed.entries) {
+            entry.feed = feed
+        }
+        const entry = feed.entries[0]
+        entry.published = Date.parse(entry.published)
+        if (isNaN(entry.published)) {
+            entry.published = Date.parse(feed.published)
+        }
+        entry.title = fts.stripHTML(entry.title)
+        entry.published = dayjs(entry.published)
+        entry.content = cutDesc(fts.stripHTML(entry.description))
+        entries.push(entry)
+        entry.feedName = name
+    }
+    entries.sort((a, b) => b.published - a.published)
+
+    globalData.openring = entries.slice(0, 6).map((post, i) => minifyHTML(globalData.templates.remoteFeedEntry({
+        ...globalData,
+        ...post,
+        i
+    })))
 }
 
 const compileCSS = async () => {
@@ -550,8 +592,8 @@ const tasks = {
     pagedeps: { deps: ["templates", "css"] },
     css: { deps: [], fn: compileCSS },
     writeBuildID: { deps: [], fn: writeBuildID },
-    index: { deps: ["openring", "pagedeps", "blog", "experiments", "images", "fetchMicroblog"], fn: index },
-    openring: { deps: [], fn: runOpenring },
+    index: { deps: ["fetchFeeds", "pagedeps", "blog", "experiments", "images", "fetchMicroblog"], fn: index },
+    fetchFeeds: { deps: ["templates"], fn: fetchFeeds },
     rss: { deps: ["blog"], fn: genRSS },
     blog: { deps: ["pagedeps"], fn: processBlog },
     fetchMicroblog: { deps: ["templates"], fn: fetchMicroblog },

src/openring.html

@@ -1,14 +0,0 @@
-<h2>From other blogs</h2>
-<section class="atl">
-    {{range .Articles}}
-    <div class="art">
-        <a class="ttl" href="{{.Link}}" target="_blank" rel="noopener">{{.Title}}</a>
-        <p class="sum">{{.Summary}}</p>
-        <small class="src">
-            via <a href="{{.SourceLink}}">{{.SourceTitle}}</a>
-        </small>
-        <small class="dat">{{.Date | datef "02/01/2006"}}</small>
-    </div>
-    {{end}}
-</section>
-<p class="atr">Generated by <a href="https://git.sr.ht/~sircmpwn/openring">openring</a></p>

src/style.sass

@@ -50,7 +50,7 @@ body
 pre, code, .deemph
     font-family: $monospace
     line-height: 1.3
-    font-size: 0.9em
+    font-size: 0.95em
 
 a
     text-decoration: none
@@ -115,7 +115,7 @@ main
         padding-left: $content-margin
         padding-right: $content-margin
 
-.blog, .experiments, .atl, .microblog
+.blog, .experiments, .ring, .microblog
     margin-left: -1rem
     margin-right: -1rem
     margin-bottom: 0
@@ -168,18 +168,6 @@ button, select, input, textarea, .textarea
     margin-top: 0.5em
     margin-bottom: 0.5em
 
-.ring
-    .art
-        flex-direction: column
-        flex: 1 1 25%
-    .sum
-        font-size: 0.8rem
-        flex: 1 1 0
-    .atr
-        text-align: right
-        font-size: 0.8rem
-        color: #555
-
 .imbox
     display: flex
     img, picture