From a81f814bf785a41f91cc949d91d16a6b77caccb0 Mon Sep 17 00:00:00 2001 From: osmarks Date: Wed, 24 Feb 2021 10:32:10 +0000 Subject: [PATCH] compiler parallelization --- package-lock.json | 49 ++++++++++++- package.json | 1 + src/index.js | 180 +++++++++++++++++++++++++++------------------- 3 files changed, 154 insertions(+), 76 deletions(-) diff --git a/package-lock.json b/package-lock.json index 32ba894..f85f44e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,14 @@ "repeat-string": "^1.5.2" } }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -126,6 +134,15 @@ "lazy-cache": "^1.0.3" } }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, "character-parser": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz", @@ -167,6 +184,19 @@ "wordwrap": "0.0.2" } }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", @@ -322,6 +352,11 @@ "function-bind": "^1.1.1" } }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -499,9 +534,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "longest": { "version": "1.0.1", @@ -779,6 +814,14 @@ "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", "integrity": "sha1-5SEekiQ2n7uB1jOi8ABE3IztrZI=" }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, "terser": { "version": "4.8.0", "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", diff --git a/package.json b/package.json index c69dc88..f96278e 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "Static site generation code for my website.", "main": "index.js", "dependencies": { + "chalk": "^4.1.0", "dayjs": "^1.8.28", "fs-extra": "^8.1.0", "gray-matter": "^4.0.2", diff --git a/src/index.js b/src/index.js index a979ba7..1d9f58b 100644 --- a/src/index.js +++ b/src/index.js @@ -15,6 +15,7 @@ const htmlMinifier = require("html-minifier").minify const terser = require("terser") const util = require("util") const childProcess = require("child_process") +const chalk = require("chalk") dayjs.extend(customParseFormat) @@ -99,12 +100,13 @@ const applyTemplate = async (template, input, getOutput, options = {}) => { return page.data } -const processExperiments = templates => { - return loadDir(experimentDir, (subdirectory, basename) => { - return applyTemplate( - templates.experiment, - path.join(subdirectory, "index.html"), - async page => { +const processExperiments = async () => { + const templates = globalData.templates + const experiments = await loadDir(experimentDir, (subdirectory, basename) => { + return applyTemplate( + templates.experiment, + path.join(subdirectory, "index.html"), + async page => { const out = path.join(outDir, page.data.slug) await fse.ensureDir(out) const allFiles = await fsp.readdir(subdirectory) @@ -112,24 +114,30 @@ const processExperiments = templates => { if (file !== "index.html") { return fse.copy(path.join(subdirectory, file), path.join(out, file)) } - })) - return path.join(out, "index.html") - }, - { processMeta: meta => { meta.slug = meta.slug || basename }}) + })) + return path.join(out, "index.html") + }, + { processMeta: meta => { meta.slug = meta.slug || basename }}) }) + console.log(chalk.yellow(`${Object.keys(experiments).length} experiments`)) + globalData.experiments = R.sortBy(x => x.title, R.values(experiments)) } -const processBlog = templates => { - return loadDir(blogDir, async (file, basename) => { +const processBlog = async () => { + const templates = globalData.templates + const blog = await loadDir(blogDir, async (file, basename) => { return applyTemplate(templates.blogPost, file, async page => { const out = path.join(outDir, page.data.slug) await fse.ensureDir(out) return path.join(out, "index.html") }, { processMeta: meta => { meta.slug = meta.slug || removeExtension(basename) }, processContent: renderMarkdown }) }) + console.log(chalk.yellow(`${Object.keys(blog).length} blog entries`)) + globalData.blog = R.sortBy(x => x.updated ? -x.updated.valueOf() : 0, R.values(blog)) } -const processErrorPages = templates => { +const processErrorPages = () => { + const templates = globalData.templates return loadDir(errorPagesDir, async (file, basename) => { return applyTemplate(templates.experiment, file, async page => { return path.join(outDir, basename) @@ -137,76 +145,102 @@ const processErrorPages = templates => { }) } -const processAssets = async templates => { - const outAssets = path.join(outDir, "assets") - await fse.ensureDir(outAssets) - - // Write out the web manifest after templating it using somewhat misapplied frontend templating stuff - const runManifest = async () => { - const m = mustache.render(await readFile(path.join(assetsDir, "manifest.webmanifest")), globalData) - fsp.writeFile(path.join(outAssets, "manifest.webmanifest"), m) - } - - const runJS = async () => { - const jsDir = path.join(assetsDir, "js") - const jsOutDir = path.join(outAssets, "js") - await Promise.all((await fsp.readdir(jsDir)).map(async file => { - const fullpath = path.join(jsDir, file) - await minifyJSFile(await readFile(fullpath), file, path.join(jsOutDir, file)) - })) - - const serviceWorker = mustache.render(await readFile(path.join(assetsDir, "sw.js")), globalData) - await minifyJSFile(serviceWorker, "sw.js", path.join(outDir, "sw.js")) - } - - const copyAsset = subpath => fse.copy(path.join(assetsDir, subpath), path.join(outAssets, subpath)) - // Directly copy images, JS, CSS - await Promise.all([ - copyAsset("images"), - copyAsset("css"), - runManifest, - runJS, - applyTemplate(templates.experiment, path.join(assetsDir, "offline.html"), () => path.join(outAssets, "offline.html"), {}) - ]) -} +const outAssets = path.join(outDir, "assets") globalData.renderDate = date => date.format("DD/MM/YYYY") -const runOpenring = async () => { - // wildly unsafe but only runs on input from me anyway - const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < openring.html` - console.log(arg) - const out = await util.promisify(childProcess.exec)(arg) - console.log(out.stderr) - return minifyHTML(out.stdout) +const writeBuildID = () => fsp.writeFile(path.join(outDir, "buildID.txt"), buildID) +const index = async () => { + const index = globalData.templates.index({ ...globalData, title: "Index", posts: globalData.blog }) + await fsp.writeFile(path.join(outDir, "index.html"), index) } - -const run = async () => { - const css = await sass.renderSync({ +const compileCSS = async () => { + const css = sass.renderSync({ data: await readFile(path.join(root, "style.sass")), outputStyle: "compressed", indentedSyntax: true }).css globalData.css = css - - const templates = await loadDir(templateDir, async fullPath => pug.compile(await readFile(fullPath), { filename: fullPath })) - const [exp, blg, opr, ..._] = await Promise.all([ - processExperiments(templates), - processBlog(templates), - runOpenring(), - processAssets(templates), - processErrorPages(templates) - ]) - const experimentsList = R.sortBy(x => x.title, R.values(exp)) - const blogList = R.sortBy(x => x.updated ? -x.updated.valueOf() : 0, R.values(blg)) - - const index = templates.index({ ...globalData, title: "Index", experiments: experimentsList, posts: blogList, openring: opr }) - await fsp.writeFile(path.join(outDir, "index.html"), index) - - const rssFeed = templates.rss({ ...globalData, items: blogList, lastUpdate: new Date() }) +} +const loadTemplates = async () => { + globalData.templates = await loadDir(templateDir, async fullPath => pug.compile(await readFile(fullPath), { filename: fullPath })) +} +const runOpenring = async () => { + // wildly unsafe but only runs on input from me anyway + const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < openring.html` + console.log(chalk.keyword("orange")("Openring:") + " " + arg) + const out = await util.promisify(childProcess.exec)(arg) + console.log(chalk.keyword("orange")("Openring:") + "\n" + out.stderr.trim()) + globalData.openring = minifyHTML(out.stdout) +} +const genRSS = async () => { + const rssFeed = globalData.templates.rss({ ...globalData, items: globalData.blog, lastUpdate: new Date() }) await fsp.writeFile(path.join(outDir, "rss.xml"), rssFeed) +} +const genManifest = async () => { + const m = mustache.render(await readFile(path.join(assetsDir, "manifest.webmanifest")), globalData) + fsp.writeFile(path.join(outAssets, "manifest.webmanifest"), m) +} +const minifyJSTask = async () => { + const jsDir = path.join(assetsDir, "js") + const jsOutDir = path.join(outAssets, "js") + await Promise.all((await fsp.readdir(jsDir)).map(async file => { + const fullpath = path.join(jsDir, file) + await minifyJSFile(await readFile(fullpath), file, path.join(jsOutDir, file)) + })) +} +const genServiceWorker = async () => { + const serviceWorker = mustache.render(await readFile(path.join(assetsDir, "sw.js")), globalData) + await minifyJSFile(serviceWorker, "sw.js", path.join(outDir, "sw.js")) +} +const copyAsset = subpath => fse.copy(path.join(assetsDir, subpath), path.join(outAssets, subpath)) - await fsp.writeFile(path.join(outDir, "buildID.txt"), buildID) +const tasks = { + errorPages: { deps: ["pagedeps"], fn: processErrorPages }, + templates: { deps: [], fn: loadTemplates }, + pagedeps: { deps: ["templates", "css"] }, + css: { deps: [], fn: compileCSS }, + writeBuildID: { deps: [], fn: writeBuildID }, + index: { deps: ["openring", "pagedeps", "blog", "experiments"], fn: index }, + openring: { deps: [], fn: runOpenring }, + rss: { deps: ["blog"], fn: genRSS }, + blog: { deps: ["pagedeps"], fn: processBlog }, + experiments: { deps: ["pagedeps"], fn: processExperiments }, + assetsDir: { deps: [], fn: () => fse.ensureDir(outAssets) }, + manifest: { deps: ["assetsDir"], fn: genManifest }, + minifyJS: { deps: ["assetsDir"], fn: minifyJSTask }, + serviceWorker: { deps: [], fn: genServiceWorker }, + images: { deps: ["assetsDir"], fn: () => copyAsset("images") }, + offlinePage: { deps: ["assetsDir", "pagedeps"], fn: () => applyTemplate(globalData.templates.experiment, path.join(assetsDir, "offline.html"), () => path.join(outAssets, "offline.html"), {}) }, + assets: { deps: ["manifest", "minifyJS", "serviceWorker", "images"] }, + main: { deps: ["writeBuildID", "index", "errorPages", "assets", "experiments", "blog", "rss"] } } -run() +const compile = async () => { + const done = new Set() + const inProgress = new Set() + const go = async finished => { + if (finished) { + inProgress.delete(finished) + done.add(finished) + console.log(`[${done.size}/${Object.keys(tasks).length}] ` + chalk.green(`Done ${finished}`)) + } + for (const [task, conf] of Object.entries(tasks)) { + if (!inProgress.has(task) && !done.has(task) && conf.deps.every(x => done.has(x))) { // dependencies now satisfied for task, execute it + inProgress.add(task) + const callback = () => go(task) + if (conf.fn) { + console.log(chalk.cyanBright(`Executing ${task}`)) + Promise.resolve(conf.fn()).then(callback, err => { + console.error(`Error in ${task}: ${err.stack}`) + process.exit(1) + }) + } else { + setImmediate(callback) + } + } + } + } + go() +} +compile() \ No newline at end of file