mirror of
https://github.com/osmarks/meme-search-engine.git
synced 2026-10-03 14:21:47 +00:00
Rewrite entire application (well, backend) in Rust and also Go
I decided I wanted to integrate the experimental OCR thing better, so I rewrote in Go and also integrated the thumbnailer. However, Go is a bad langauge and I only used it out of spite. It turned out to have a very hard-to-fix memory leak due to some unclear interaction between libvips and both sets of bindings I tried, so I had Claude-3 transpile it to Rust then spent a while fixing the several mistakes it made and making tweaks. The new Rust version works, although I need to actually do something with the OCR data and make the index queryable concurrently.
This commit is contained in:
+17
-10
@@ -62,6 +62,8 @@
|
||||
|
||||
.result
|
||||
border: 1px solid gray
|
||||
*
|
||||
display: block
|
||||
.result img
|
||||
width: 100%
|
||||
</style>
|
||||
@@ -109,17 +111,22 @@
|
||||
<Loading />
|
||||
{/if}
|
||||
{#if results}
|
||||
{#if displayedResults.length === 0}
|
||||
No results. Wait for index rebuild.
|
||||
{/if}
|
||||
<Masonry bind:refreshLayout={refreshLayout} colWidth="minmax(Min(20em, 100%), 1fr)" items={displayedResults}>
|
||||
{#each displayedResults as result}
|
||||
{#key result.file}
|
||||
<div class="result">
|
||||
<a href={util.getURL(result.file)}>
|
||||
<a href={util.getURL(result)}>
|
||||
<picture>
|
||||
{#if util.hasThumbnails(result.file)}
|
||||
<source srcset={util.thumbnailPath(result.file, "avif-lq") + ", " + util.thumbnailPath(result.file, "avif-hq") + " 2x"} type="image/avif" />
|
||||
<source srcset={util.thumbnailPath(result.file, "jpeg-800") + " 800w, " + util.thumbnailPath(result.file, "jpeg-fullscale")} type="image/jpeg" />
|
||||
{#if util.hasFormat(results, result, "avifl")}
|
||||
<source srcset={util.thumbnailURL(results, result, "avifl") + (util.hasFormat(results, result, "avifh") ? ", " + util.thumbnailURL(results, result, "avifh") + " 2x" : "")} type="image/avif" />
|
||||
{/if}
|
||||
<img src={util.getURL(result.file)} on:load={updateCounter} on:error={updateCounter} alt={result.caption || result.file}>
|
||||
{#if util.hasFormat(results, result, "jpegl")}
|
||||
<source srcset={util.thumbnailURL(results, result, "jpegl") + (util.hasFormat(results, result, "jpegh") ? ", " + util.thumbnailURL(results, result, "jpegh") + " 2x" : "")} type="image/jpeg" />
|
||||
{/if}
|
||||
<img src={util.getURL(result)} on:load={updateCounter} on:error={updateCounter} alt={result[1]}>
|
||||
</picture>
|
||||
</a>
|
||||
</div>
|
||||
@@ -171,9 +178,7 @@
|
||||
let displayedResults = []
|
||||
const runSearch = async () => {
|
||||
if (!resultPromise) {
|
||||
let args = {}
|
||||
args.text = queryTerms.filter(x => x.type === "text" && x.text).map(({ text, weight, sign }) => [ text, weight * { "+": 1, "-": -1 }[sign] ])
|
||||
args.images = queryTerms.filter(x => x.type === "image").map(({ imageData, weight, sign }) => [ imageData, weight * { "+": 1, "-": -1 }[sign] ])
|
||||
let args = {"terms": queryTerms.map(x => ({ image: x.imageData, text: x.text, weight: x.weight * { "+": 1, "-": -1 }[x.sign] }))}
|
||||
resultPromise = util.doQuery(args).then(res => {
|
||||
error = null
|
||||
results = res
|
||||
@@ -181,7 +186,8 @@
|
||||
displayedResults = []
|
||||
pendingImageLoads = 0
|
||||
for (let i = 0; i < chunkSize; i++) {
|
||||
displayedResults.push(results[i])
|
||||
if (i >= results.matches.length) break
|
||||
displayedResults.push(results.matches[i])
|
||||
pendingImageLoads += 1
|
||||
}
|
||||
redrawGrid()
|
||||
@@ -195,7 +201,8 @@
|
||||
if (window.scrollY + window.innerHeight < heightThreshold) return;
|
||||
let init = displayedResults.length
|
||||
for (let i = 0; i < chunkSize; i++) {
|
||||
displayedResults.push(results[init + i])
|
||||
if (init + i >= results.matches.length) break
|
||||
displayedResults.push(results.matches[init + i])
|
||||
pendingImageLoads += 1
|
||||
}
|
||||
displayedResults = displayedResults
|
||||
|
||||
@@ -7,7 +7,7 @@ esbuild
|
||||
.build({
|
||||
entryPoints: [path.join(__dirname, "app.js")],
|
||||
bundle: true,
|
||||
minify: true,
|
||||
minify: false,
|
||||
outfile: path.join(__dirname, "../static/app.js"),
|
||||
plugins: [sveltePlugin({
|
||||
preprocess: {
|
||||
|
||||
+7
-10
@@ -1,7 +1,8 @@
|
||||
import * as config from "../../frontend_config.json"
|
||||
import * as backendConfig from "../../mse_config.json"
|
||||
import * as formats from "../../formats.json"
|
||||
|
||||
export const getURL = x => config.image_path + x
|
||||
export const getURL = x => config.image_path + x[1]
|
||||
|
||||
export const doQuery = args => fetch(config.backend_url, {
|
||||
method: "POST",
|
||||
@@ -11,15 +12,11 @@ export const doQuery = args => fetch(config.backend_url, {
|
||||
body: JSON.stringify(args)
|
||||
}).then(x => x.json())
|
||||
|
||||
const filesafeCharset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
|
||||
export const thumbnailPath = (originalPath, format) => {
|
||||
const extension = formats.formats[format][0]
|
||||
// Python and JS have minor differences in string handling wrt. astral characters which could result in incorrect quantities of dashes. Fortunately, Array.from handles this correctly.
|
||||
return config.thumb_path + `${Array.from(originalPath).map(x => filesafeCharset.includes(x) ? x : "_").join("")}.${format}${extension}`
|
||||
export const hasFormat = (results, result, format) => {
|
||||
return result[3] && (1 << results.formats.indexOf(format)) !== 0
|
||||
}
|
||||
|
||||
const thumbedExtensions = formats.extensions
|
||||
export const hasThumbnails = t => {
|
||||
const parts = t.split(".")
|
||||
return thumbedExtensions.includes("." + parts[parts.length - 1])
|
||||
export const thumbnailURL = (results, result, format) => {
|
||||
console.log("RES", results)
|
||||
return `${config.thumb_path}${result[2]}${format}.${results.extensions[format]}`
|
||||
}
|
||||
Reference in New Issue
Block a user