website/src/index.js

const fsp = require("fs").promises
const fse = require("fs-extra")
const MarkdownIt = require("markdown-it")
const pug = require("pug")
const path = require("path")
const matter = require("gray-matter")
const mustache = require("mustache")
const globalData = require("./global.json")
const sass = require("sass")
const R = require("ramda")
const dayjs = require("dayjs")
const customParseFormat = require("dayjs/plugin/customParseFormat")
const nanoid = require("nanoid")
const htmlMinifier = require("html-minifier").minify
const terser = require("terser")
const util = require("util")
const childProcess = require("child_process")

dayjs.extend(customParseFormat)

const root = path.join(__dirname, "..")
const templateDir = path.join(root, "templates")
const experimentDir = path.join(root, "experiments")
const blogDir = path.join(root, "blog")
const errorPagesDir = path.join(root, "error")
const assetsDir = path.join(root, "assets")
const outDir = path.join(root, "out")

const buildID = nanoid()
globalData.buildID = buildID

const removeExtension = x => x.replace(/\.[^/.]+$/, "")

const readFile = path => fsp.readFile(path, { encoding: "utf8" })
const md = new MarkdownIt()
const minifyHTML = x => htmlMinifier(x, {
    collapseWhitespace: true,
    sortAttributes: true,
    sortClassName: true,
    removeRedundantAttributes: true,
    removeAttributeQuotes: true,
    conservativeCollapse: true,
    collapseBooleanAttributes: true
})
const renderMarkdown = x => md.render(x)
// basically just whitespace removal - fast and still pretty good versus unminified code
const minifyJS = (x, filename) => {
    const res = terser.minify(x, {
        mangle: true,
        compress: true,
        sourceMap: {
            filename,
            url: filename + ".map"
        }
    })
    if (res.warnings) {
        for (const warn of res.warnings) {
            console.warn("MINIFY WARNING:", warn)
        }
    }
    if (res.error) {
        console.warn("MINIFY ERROR:", res.error)
        throw new Error("could not minify " + filename)
    }
    return res
}
const minifyJSFile = (incode, filename, out) => {
    const res = minifyJS(incode, filename)
    return Promise.all([ fsp.writeFile(out, res.code), fsp.writeFile(out + ".map", res.map) ])
}

const parseFrontMatter = content => {
    const raw = matter(content)
    if (raw.data.updated) {
        raw.data.updated = dayjs(raw.data.updated, "DD/MM/YYYY")
    }
    if (raw.data.created) {
        raw.data.created = dayjs(raw.data.created, "DD/MM/YYYY")
    }
    if (raw.data.created && !raw.data.updated) { raw.data.updated = raw.data.created }
    return raw
}

const loadDir = async (dir, func) => {
    const files = await fsp.readdir(dir)
    const out = {}
    await Promise.all(files.map(async file => {
        out[removeExtension(file)] = await func(path.join(dir, file), file)
    }))
    return out
}

const applyTemplate = async (template, input, getOutput, options = {}) => {
    const page = parseFrontMatter(await readFile(input))
    if (options.processMeta) { options.processMeta(page.data) }
    if (options.processContent) { page.content = options.processContent(page.content) }
    const rendered = template({ ...globalData, ...page.data, content: page.content })
    await fsp.writeFile(await getOutput(page), minifyHTML(rendered))
    return page.data
}

const processExperiments = templates => {
    return loadDir(experimentDir, (subdirectory, basename) => {
       return applyTemplate(
           templates.experiment,
           path.join(subdirectory, "index.html"),
           async page => {
                const out = path.join(outDir, page.data.slug)
                await fse.ensureDir(out)
                const allFiles = await fsp.readdir(subdirectory)
                await Promise.all(allFiles.map(file => {
                    if (file !== "index.html") {
                        return fse.copy(path.join(subdirectory, file), path.join(out, file))
                    }
               }))
               return path.join(out, "index.html")
           },
           { processMeta: meta => { meta.slug = meta.slug || basename }})
    })
}

const processBlog = templates => {
    return loadDir(blogDir, async (file, basename) => {
        return applyTemplate(templates.blogPost, file, async page => {
            const out = path.join(outDir, page.data.slug)
            await fse.ensureDir(out)
            return path.join(out, "index.html")
        }, { processMeta: meta => { meta.slug = meta.slug || removeExtension(basename) }, processContent: renderMarkdown })
    })
}

const processErrorPages = templates => {
    return loadDir(errorPagesDir, async (file, basename) => {
        return applyTemplate(templates.experiment, file, async page => {
            return path.join(outDir, basename)
        })
    })
}

const processAssets = async templates => {
    const outAssets = path.join(outDir, "assets")
    await fse.ensureDir(outAssets)

    // Write out the web manifest after templating it using somewhat misapplied frontend templating stuff 
    const runManifest = async () => {
        const m = mustache.render(await readFile(path.join(assetsDir, "manifest.webmanifest")), globalData)
        fsp.writeFile(path.join(outAssets, "manifest.webmanifest"), m)
    }

    const runJS = async () => {
        const jsDir = path.join(assetsDir, "js")
        const jsOutDir = path.join(outAssets, "js")
        await Promise.all((await fsp.readdir(jsDir)).map(async file => {
            const fullpath = path.join(jsDir, file)
            await minifyJSFile(await readFile(fullpath), file, path.join(jsOutDir, file))
        }))

        const serviceWorker = mustache.render(await readFile(path.join(assetsDir, "sw.js")), globalData)
        await minifyJSFile(serviceWorker, "sw.js", path.join(outDir, "sw.js"))
    }

    const copyAsset = subpath => fse.copy(path.join(assetsDir, subpath), path.join(outAssets, subpath))
    // Directly copy images, JS, CSS
    await Promise.all([
        copyAsset("images"), 
        copyAsset("css"), 
        runManifest,
        runJS,
        applyTemplate(templates.experiment, path.join(assetsDir, "offline.html"), () => path.join(outAssets, "offline.html"), {})
    ])
}

globalData.renderDate = date => date.format("DD/MM/YYYY")

const runOpenring = async () => {
    // wildly unsafe but only runs on input from me anyway
    const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < openring.html`
    console.log(arg)
    const out = await util.promisify(childProcess.exec)(arg)
    console.log(out.stderr)
    return minifyHTML(out.stdout)
}

const run = async () => {
    const css = await sass.renderSync({
        data: await readFile(path.join(root, "style.sass")),
        outputStyle: "compressed",
        indentedSyntax: true
    }).css
    globalData.css = css

    const templates = await loadDir(templateDir, async fullPath => pug.compile(await readFile(fullPath), { filename: fullPath }))
    const [exp, blg, opr, ..._] = await Promise.all([
        processExperiments(templates), 
        processBlog(templates),
        runOpenring(),
        processAssets(templates), 
        processErrorPages(templates)
    ])
    const experimentsList = R.sortBy(x => x.title, R.values(exp))
    const blogList = R.sortBy(x => x.updated ? -x.updated.valueOf() : 0, R.values(blg))

    const index = templates.index({ ...globalData, title: "Index", experiments: experimentsList, posts: blogList, openring: opr })
    await fsp.writeFile(path.join(outDir, "index.html"), index)

    const rssFeed = templates.rss({ ...globalData, items: blogList, lastUpdate: new Date() })
    await fsp.writeFile(path.join(outDir, "rss.xml"), rssFeed)

    await fsp.writeFile(path.join(outDir, "buildID.txt"), buildID)
}

run()