diff --git a/README.md b/README.md index 9c29bbf..f85979f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ They say a picture is worth a thousand words. Unfortunately, many (most?) sets o ## Setup +New: somewhat tested Dockerized setup under /docker/. Note that you need a model like https://huggingface.co/timm/ViT-SO400M-14-SigLIP-384 (for timm), not one packaged for other libraries. + This is untested. It might work. The new Rust version simplifies some steps (it integrates its own thumbnailing). * Serve your meme library from a static webserver. diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..c4d6810 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,92 @@ +# Base Image +FROM debian:trixie-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + git \ + wget \ + build-essential \ + python3 \ + python3-pip \ + nginx \ + curl \ + pkg-config \ + libssl-dev \ + nasm \ + meson \ + ninja-build \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libswscale-dev \ + libswresample-dev \ + libpostproc-dev \ + libsqlite3-dev \ + libzstd-dev \ + libjpeg-dev \ + libpng-dev \ + libwebp-dev \ + libavdevice-dev \ + libclang-dev \ + libopenblas-dev \ + nodejs \ + npm \ + cmake \ + rustup \ + dav1d \ + libdav1d-dev \ + && apt-get clean + +# Set environment variables for Rust +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install nightly Rust and set it as default +RUN rustup install nightly \ + && rustup default nightly + +# Update Rust to the latest version and confirm nightly is active +RUN rustup update && rustc --version + +# Build and install FAISS from source +RUN git clone https://github.com/Enet4/faiss /tmp/faiss -b c_api_head \ + && cd /tmp/faiss \ + && sed -i "14i #include " faiss/Index.h \ + && cmake -B build -DFAISS_ENABLE_PYTHON=OFF -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=OFF . \ + && cmake --build build --target install \ + && cp build/c_api/libfaiss_c.so /usr/local/lib/ \ + && cp build/faiss/libfaiss.so /usr/local/lib \ + && rm -rf /tmp/faiss + +# Set environment variables for library paths +ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig:/usr/lib/x86_64-linux-gnu/pkgconfig" +ENV LD_LIBRARY_PATH="/usr/local/lib:${LD_LIBRARY_PATH}" + +# Set working directory +WORKDIR /app + +# Clone the repository +RUN git clone https://github.com/osmarks/meme-search-engine.git . && git checkout 5121de63890ca5e3024a4a630385a274acbb510a + +# Build Rust backend +RUN cargo +nightly build --release --bin meme-search-engine + +# Install Python dependencies +RUN pip install -r requirements.txt --break-system-packages + +COPY config/frontend_config.json /app + +# Build frontend +RUN cd clipfront2 && npm install && node src/build.js + +# Configure nginx +RUN rm /etc/nginx/sites-enabled/default +COPY config/nginx.conf /etc/nginx/sites-enabled/default + +# Expose necessary ports +EXPOSE 80 1707 1708 + +# Start all services +CMD ["bash", "-c", "\ + python3 clip_server.py /app/config/clip_server_config.json & \ + /app/target/release/meme-search-engine /app/config/backend_config.json & \ + nginx -g 'daemon off;'"] diff --git a/docker/config/backend_config.json b/docker/config/backend_config.json new file mode 100644 index 0000000..612294f --- /dev/null +++ b/docker/config/backend_config.json @@ -0,0 +1,9 @@ +{ + "clip_server": "http://localhost:1708", + "db_path": "/app/db/data.sqlite3", + "port": 1707, + "files": "/data/memes/", + "enable_ocr": false, + "thumbs_path": "/data/thumbs", + "enable_thumbs": true +} diff --git a/docker/config/clip_server_config.json b/docker/config/clip_server_config.json new file mode 100644 index 0000000..52174aa --- /dev/null +++ b/docker/config/clip_server_config.json @@ -0,0 +1,8 @@ +{ + "model": "ViT-SO400M-14-SigLIP-384", + "model_path": "/app/siglip_model/open_clip_model.safetensors", + "model_name": "siglip-so400m/14@384", + "max_batch_size": 128, + "port": 1708, + "device": "cpu" +} diff --git a/docker/config/frontend_config.json b/docker/config/frontend_config.json new file mode 100644 index 0000000..f85abd6 --- /dev/null +++ b/docker/config/frontend_config.json @@ -0,0 +1,8 @@ +{ + "backend_url": "/backend", + "image_path": "/memes/", + "thumb_path": "/thumbs/", + "name": "Meme Search Engine", + "description": "Local instance for something", + "about_html": "hello" +} diff --git a/docker/config/nginx.conf b/docker/config/nginx.conf new file mode 100644 index 0000000..1458012 --- /dev/null +++ b/docker/config/nginx.conf @@ -0,0 +1,20 @@ +server { + listen 80; + + location / { + root /app/clipfront2/static; + index index.html; + } + + location /memes/ { + alias /data/memes/; + } + + location /thumbs/ { + alias /data/thumbs/; + } + + location /backend { + proxy_pass http://localhost:1707/; + } +} diff --git a/docker/config/thumbnailer_config.json b/docker/config/thumbnailer_config.json new file mode 100644 index 0000000..f7cd952 --- /dev/null +++ b/docker/config/thumbnailer_config.json @@ -0,0 +1,5 @@ +{ + "database": "/app/db/thumb.sqlite3", + "input": "/data/memes", + "output": "/data/memes/thumbs" +} \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..989debc --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,17 @@ +services: + meme-engine: + build: + context: . + dockerfile: Dockerfile + ports: + - "8001:80" # Frontend + - "17080:1708" # Clip server + volumes: + - ./config:/app/config # Configuration files + - ./memes:/data/memes # Meme storage + - ./thumbs:/data/thumbs + - ./db:/app/db # Databases + - ./siglip_model:/app/siglip_model # Model files + environment: + HTTPS_PROXY: "http://100.64.0.2:8888" + # command: tail -f /dev/null diff --git a/docker/memes/.gitignore b/docker/memes/.gitignore new file mode 100644 index 0000000..67c04e4 --- /dev/null +++ b/docker/memes/.gitignore @@ -0,0 +1,2 @@ +*.* +!.gitignore diff --git a/docker/thumbs/.gitkeep b/docker/thumbs/.gitkeep new file mode 100644 index 0000000..e69de29