mirror of
https://github.com/osmarks/website
synced 2024-12-24 17:10:32 +00:00
compiler parallelization
This commit is contained in:
parent
0748ec1289
commit
a81f814bf7
49
package-lock.json
generated
49
package-lock.json
generated
@ -47,6 +47,14 @@
|
|||||||
"repeat-string": "^1.5.2"
|
"repeat-string": "^1.5.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"ansi-styles": {
|
||||||
|
"version": "4.3.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||||
|
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||||
|
"requires": {
|
||||||
|
"color-convert": "^2.0.1"
|
||||||
|
}
|
||||||
|
},
|
||||||
"anymatch": {
|
"anymatch": {
|
||||||
"version": "3.1.1",
|
"version": "3.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
|
||||||
@ -126,6 +134,15 @@
|
|||||||
"lazy-cache": "^1.0.3"
|
"lazy-cache": "^1.0.3"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"chalk": {
|
||||||
|
"version": "4.1.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz",
|
||||||
|
"integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==",
|
||||||
|
"requires": {
|
||||||
|
"ansi-styles": "^4.1.0",
|
||||||
|
"supports-color": "^7.1.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"character-parser": {
|
"character-parser": {
|
||||||
"version": "2.2.0",
|
"version": "2.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz",
|
||||||
@ -167,6 +184,19 @@
|
|||||||
"wordwrap": "0.0.2"
|
"wordwrap": "0.0.2"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"color-convert": {
|
||||||
|
"version": "2.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||||
|
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||||
|
"requires": {
|
||||||
|
"color-name": "~1.1.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"color-name": {
|
||||||
|
"version": "1.1.4",
|
||||||
|
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||||
|
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
|
||||||
|
},
|
||||||
"commander": {
|
"commander": {
|
||||||
"version": "2.20.3",
|
"version": "2.20.3",
|
||||||
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
|
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
|
||||||
@ -322,6 +352,11 @@
|
|||||||
"function-bind": "^1.1.1"
|
"function-bind": "^1.1.1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"has-flag": {
|
||||||
|
"version": "4.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||||
|
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="
|
||||||
|
},
|
||||||
"he": {
|
"he": {
|
||||||
"version": "1.2.0",
|
"version": "1.2.0",
|
||||||
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
||||||
@ -499,9 +534,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"lodash": {
|
"lodash": {
|
||||||
"version": "4.17.15",
|
"version": "4.17.21",
|
||||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
|
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
|
||||||
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
|
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
|
||||||
},
|
},
|
||||||
"longest": {
|
"longest": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
@ -779,6 +814,14 @@
|
|||||||
"resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz",
|
||||||
"integrity": "sha1-5SEekiQ2n7uB1jOi8ABE3IztrZI="
|
"integrity": "sha1-5SEekiQ2n7uB1jOi8ABE3IztrZI="
|
||||||
},
|
},
|
||||||
|
"supports-color": {
|
||||||
|
"version": "7.2.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||||
|
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||||
|
"requires": {
|
||||||
|
"has-flag": "^4.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"terser": {
|
"terser": {
|
||||||
"version": "4.8.0",
|
"version": "4.8.0",
|
||||||
"resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz",
|
"resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz",
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
"description": "Static site generation code for my website.",
|
"description": "Static site generation code for my website.",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"chalk": "^4.1.0",
|
||||||
"dayjs": "^1.8.28",
|
"dayjs": "^1.8.28",
|
||||||
"fs-extra": "^8.1.0",
|
"fs-extra": "^8.1.0",
|
||||||
"gray-matter": "^4.0.2",
|
"gray-matter": "^4.0.2",
|
||||||
|
152
src/index.js
152
src/index.js
@ -15,6 +15,7 @@ const htmlMinifier = require("html-minifier").minify
|
|||||||
const terser = require("terser")
|
const terser = require("terser")
|
||||||
const util = require("util")
|
const util = require("util")
|
||||||
const childProcess = require("child_process")
|
const childProcess = require("child_process")
|
||||||
|
const chalk = require("chalk")
|
||||||
|
|
||||||
dayjs.extend(customParseFormat)
|
dayjs.extend(customParseFormat)
|
||||||
|
|
||||||
@ -99,8 +100,9 @@ const applyTemplate = async (template, input, getOutput, options = {}) => {
|
|||||||
return page.data
|
return page.data
|
||||||
}
|
}
|
||||||
|
|
||||||
const processExperiments = templates => {
|
const processExperiments = async () => {
|
||||||
return loadDir(experimentDir, (subdirectory, basename) => {
|
const templates = globalData.templates
|
||||||
|
const experiments = await loadDir(experimentDir, (subdirectory, basename) => {
|
||||||
return applyTemplate(
|
return applyTemplate(
|
||||||
templates.experiment,
|
templates.experiment,
|
||||||
path.join(subdirectory, "index.html"),
|
path.join(subdirectory, "index.html"),
|
||||||
@ -117,19 +119,25 @@ const processExperiments = templates => {
|
|||||||
},
|
},
|
||||||
{ processMeta: meta => { meta.slug = meta.slug || basename }})
|
{ processMeta: meta => { meta.slug = meta.slug || basename }})
|
||||||
})
|
})
|
||||||
|
console.log(chalk.yellow(`${Object.keys(experiments).length} experiments`))
|
||||||
|
globalData.experiments = R.sortBy(x => x.title, R.values(experiments))
|
||||||
}
|
}
|
||||||
|
|
||||||
const processBlog = templates => {
|
const processBlog = async () => {
|
||||||
return loadDir(blogDir, async (file, basename) => {
|
const templates = globalData.templates
|
||||||
|
const blog = await loadDir(blogDir, async (file, basename) => {
|
||||||
return applyTemplate(templates.blogPost, file, async page => {
|
return applyTemplate(templates.blogPost, file, async page => {
|
||||||
const out = path.join(outDir, page.data.slug)
|
const out = path.join(outDir, page.data.slug)
|
||||||
await fse.ensureDir(out)
|
await fse.ensureDir(out)
|
||||||
return path.join(out, "index.html")
|
return path.join(out, "index.html")
|
||||||
}, { processMeta: meta => { meta.slug = meta.slug || removeExtension(basename) }, processContent: renderMarkdown })
|
}, { processMeta: meta => { meta.slug = meta.slug || removeExtension(basename) }, processContent: renderMarkdown })
|
||||||
})
|
})
|
||||||
|
console.log(chalk.yellow(`${Object.keys(blog).length} blog entries`))
|
||||||
|
globalData.blog = R.sortBy(x => x.updated ? -x.updated.valueOf() : 0, R.values(blog))
|
||||||
}
|
}
|
||||||
|
|
||||||
const processErrorPages = templates => {
|
const processErrorPages = () => {
|
||||||
|
const templates = globalData.templates
|
||||||
return loadDir(errorPagesDir, async (file, basename) => {
|
return loadDir(errorPagesDir, async (file, basename) => {
|
||||||
return applyTemplate(templates.experiment, file, async page => {
|
return applyTemplate(templates.experiment, file, async page => {
|
||||||
return path.join(outDir, basename)
|
return path.join(outDir, basename)
|
||||||
@ -137,76 +145,102 @@ const processErrorPages = templates => {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
const processAssets = async templates => {
|
|
||||||
const outAssets = path.join(outDir, "assets")
|
const outAssets = path.join(outDir, "assets")
|
||||||
await fse.ensureDir(outAssets)
|
|
||||||
|
|
||||||
// Write out the web manifest after templating it using somewhat misapplied frontend templating stuff
|
globalData.renderDate = date => date.format("DD/MM/YYYY")
|
||||||
const runManifest = async () => {
|
|
||||||
|
const writeBuildID = () => fsp.writeFile(path.join(outDir, "buildID.txt"), buildID)
|
||||||
|
const index = async () => {
|
||||||
|
const index = globalData.templates.index({ ...globalData, title: "Index", posts: globalData.blog })
|
||||||
|
await fsp.writeFile(path.join(outDir, "index.html"), index)
|
||||||
|
}
|
||||||
|
const compileCSS = async () => {
|
||||||
|
const css = sass.renderSync({
|
||||||
|
data: await readFile(path.join(root, "style.sass")),
|
||||||
|
outputStyle: "compressed",
|
||||||
|
indentedSyntax: true
|
||||||
|
}).css
|
||||||
|
globalData.css = css
|
||||||
|
}
|
||||||
|
const loadTemplates = async () => {
|
||||||
|
globalData.templates = await loadDir(templateDir, async fullPath => pug.compile(await readFile(fullPath), { filename: fullPath }))
|
||||||
|
}
|
||||||
|
const runOpenring = async () => {
|
||||||
|
// wildly unsafe but only runs on input from me anyway
|
||||||
|
const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < openring.html`
|
||||||
|
console.log(chalk.keyword("orange")("Openring:") + " " + arg)
|
||||||
|
const out = await util.promisify(childProcess.exec)(arg)
|
||||||
|
console.log(chalk.keyword("orange")("Openring:") + "\n" + out.stderr.trim())
|
||||||
|
globalData.openring = minifyHTML(out.stdout)
|
||||||
|
}
|
||||||
|
const genRSS = async () => {
|
||||||
|
const rssFeed = globalData.templates.rss({ ...globalData, items: globalData.blog, lastUpdate: new Date() })
|
||||||
|
await fsp.writeFile(path.join(outDir, "rss.xml"), rssFeed)
|
||||||
|
}
|
||||||
|
const genManifest = async () => {
|
||||||
const m = mustache.render(await readFile(path.join(assetsDir, "manifest.webmanifest")), globalData)
|
const m = mustache.render(await readFile(path.join(assetsDir, "manifest.webmanifest")), globalData)
|
||||||
fsp.writeFile(path.join(outAssets, "manifest.webmanifest"), m)
|
fsp.writeFile(path.join(outAssets, "manifest.webmanifest"), m)
|
||||||
}
|
}
|
||||||
|
const minifyJSTask = async () => {
|
||||||
const runJS = async () => {
|
|
||||||
const jsDir = path.join(assetsDir, "js")
|
const jsDir = path.join(assetsDir, "js")
|
||||||
const jsOutDir = path.join(outAssets, "js")
|
const jsOutDir = path.join(outAssets, "js")
|
||||||
await Promise.all((await fsp.readdir(jsDir)).map(async file => {
|
await Promise.all((await fsp.readdir(jsDir)).map(async file => {
|
||||||
const fullpath = path.join(jsDir, file)
|
const fullpath = path.join(jsDir, file)
|
||||||
await minifyJSFile(await readFile(fullpath), file, path.join(jsOutDir, file))
|
await minifyJSFile(await readFile(fullpath), file, path.join(jsOutDir, file))
|
||||||
}))
|
}))
|
||||||
|
}
|
||||||
|
const genServiceWorker = async () => {
|
||||||
const serviceWorker = mustache.render(await readFile(path.join(assetsDir, "sw.js")), globalData)
|
const serviceWorker = mustache.render(await readFile(path.join(assetsDir, "sw.js")), globalData)
|
||||||
await minifyJSFile(serviceWorker, "sw.js", path.join(outDir, "sw.js"))
|
await minifyJSFile(serviceWorker, "sw.js", path.join(outDir, "sw.js"))
|
||||||
}
|
}
|
||||||
|
|
||||||
const copyAsset = subpath => fse.copy(path.join(assetsDir, subpath), path.join(outAssets, subpath))
|
const copyAsset = subpath => fse.copy(path.join(assetsDir, subpath), path.join(outAssets, subpath))
|
||||||
// Directly copy images, JS, CSS
|
|
||||||
await Promise.all([
|
const tasks = {
|
||||||
copyAsset("images"),
|
errorPages: { deps: ["pagedeps"], fn: processErrorPages },
|
||||||
copyAsset("css"),
|
templates: { deps: [], fn: loadTemplates },
|
||||||
runManifest,
|
pagedeps: { deps: ["templates", "css"] },
|
||||||
runJS,
|
css: { deps: [], fn: compileCSS },
|
||||||
applyTemplate(templates.experiment, path.join(assetsDir, "offline.html"), () => path.join(outAssets, "offline.html"), {})
|
writeBuildID: { deps: [], fn: writeBuildID },
|
||||||
])
|
index: { deps: ["openring", "pagedeps", "blog", "experiments"], fn: index },
|
||||||
|
openring: { deps: [], fn: runOpenring },
|
||||||
|
rss: { deps: ["blog"], fn: genRSS },
|
||||||
|
blog: { deps: ["pagedeps"], fn: processBlog },
|
||||||
|
experiments: { deps: ["pagedeps"], fn: processExperiments },
|
||||||
|
assetsDir: { deps: [], fn: () => fse.ensureDir(outAssets) },
|
||||||
|
manifest: { deps: ["assetsDir"], fn: genManifest },
|
||||||
|
minifyJS: { deps: ["assetsDir"], fn: minifyJSTask },
|
||||||
|
serviceWorker: { deps: [], fn: genServiceWorker },
|
||||||
|
images: { deps: ["assetsDir"], fn: () => copyAsset("images") },
|
||||||
|
offlinePage: { deps: ["assetsDir", "pagedeps"], fn: () => applyTemplate(globalData.templates.experiment, path.join(assetsDir, "offline.html"), () => path.join(outAssets, "offline.html"), {}) },
|
||||||
|
assets: { deps: ["manifest", "minifyJS", "serviceWorker", "images"] },
|
||||||
|
main: { deps: ["writeBuildID", "index", "errorPages", "assets", "experiments", "blog", "rss"] }
|
||||||
}
|
}
|
||||||
|
|
||||||
globalData.renderDate = date => date.format("DD/MM/YYYY")
|
const compile = async () => {
|
||||||
|
const done = new Set()
|
||||||
const runOpenring = async () => {
|
const inProgress = new Set()
|
||||||
// wildly unsafe but only runs on input from me anyway
|
const go = async finished => {
|
||||||
const arg = `./openring -n6 ${globalData.feeds.map(x => '-s "' + x + '"').join(" ")} < openring.html`
|
if (finished) {
|
||||||
console.log(arg)
|
inProgress.delete(finished)
|
||||||
const out = await util.promisify(childProcess.exec)(arg)
|
done.add(finished)
|
||||||
console.log(out.stderr)
|
console.log(`[${done.size}/${Object.keys(tasks).length}] ` + chalk.green(`Done ${finished}`))
|
||||||
return minifyHTML(out.stdout)
|
|
||||||
}
|
}
|
||||||
|
for (const [task, conf] of Object.entries(tasks)) {
|
||||||
const run = async () => {
|
if (!inProgress.has(task) && !done.has(task) && conf.deps.every(x => done.has(x))) { // dependencies now satisfied for task, execute it
|
||||||
const css = await sass.renderSync({
|
inProgress.add(task)
|
||||||
data: await readFile(path.join(root, "style.sass")),
|
const callback = () => go(task)
|
||||||
outputStyle: "compressed",
|
if (conf.fn) {
|
||||||
indentedSyntax: true
|
console.log(chalk.cyanBright(`Executing ${task}`))
|
||||||
}).css
|
Promise.resolve(conf.fn()).then(callback, err => {
|
||||||
globalData.css = css
|
console.error(`Error in ${task}: ${err.stack}`)
|
||||||
|
process.exit(1)
|
||||||
const templates = await loadDir(templateDir, async fullPath => pug.compile(await readFile(fullPath), { filename: fullPath }))
|
})
|
||||||
const [exp, blg, opr, ..._] = await Promise.all([
|
} else {
|
||||||
processExperiments(templates),
|
setImmediate(callback)
|
||||||
processBlog(templates),
|
|
||||||
runOpenring(),
|
|
||||||
processAssets(templates),
|
|
||||||
processErrorPages(templates)
|
|
||||||
])
|
|
||||||
const experimentsList = R.sortBy(x => x.title, R.values(exp))
|
|
||||||
const blogList = R.sortBy(x => x.updated ? -x.updated.valueOf() : 0, R.values(blg))
|
|
||||||
|
|
||||||
const index = templates.index({ ...globalData, title: "Index", experiments: experimentsList, posts: blogList, openring: opr })
|
|
||||||
await fsp.writeFile(path.join(outDir, "index.html"), index)
|
|
||||||
|
|
||||||
const rssFeed = templates.rss({ ...globalData, items: blogList, lastUpdate: new Date() })
|
|
||||||
await fsp.writeFile(path.join(outDir, "rss.xml"), rssFeed)
|
|
||||||
|
|
||||||
await fsp.writeFile(path.join(outDir, "buildID.txt"), buildID)
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
run()
|
}
|
||||||
|
}
|
||||||
|
go()
|
||||||
|
}
|
||||||
|
compile()
|
Loading…
Reference in New Issue
Block a user