mirror of
https://github.com/osmarks/meme-search-engine.git
synced 2024-11-10 22:09:54 +00:00
video parsing basics
This commit is contained in:
parent
129b769a56
commit
5eae8674ce
99
Cargo.lock
generated
99
Cargo.lock
generated
@ -258,6 +258,26 @@ version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.69.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
|
||||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"itertools",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"syn 2.0.65",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit_field"
|
||||
version = "0.10.2"
|
||||
@ -356,6 +376,15 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-expr"
|
||||
version = "0.15.8"
|
||||
@ -386,6 +415,17 @@ dependencies = [
|
||||
"windows-targets 0.52.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a483f3cbf7cec2e153d424d0e92329d816becc6421389bd494375c6065921b9b"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "color_quant"
|
||||
version = "1.1.0"
|
||||
@ -688,6 +728,30 @@ dependencies = [
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ffmpeg-sys-the-third"
|
||||
version = "2.0.0+ffmpeg-7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a82bfdb0a7925996707f0a7dc37b2f3251ff5a15d26e78c586adb60c240dedc5"
|
||||
dependencies = [
|
||||
"bindgen",
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ffmpeg-the-third"
|
||||
version = "2.0.1+ffmpeg-7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4aa99eb55979d5c1db3b0b7a807a5e50dda07f5f6c2dbc6e9b50c205f611646"
|
||||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
"ffmpeg-sys-the-third",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "finl_unicode"
|
||||
version = "1.2.0"
|
||||
@ -866,6 +930,12 @@ version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.5"
|
||||
@ -1266,6 +1336,12 @@ dependencies = [
|
||||
"spin 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazycell"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||
|
||||
[[package]]
|
||||
name = "lebe"
|
||||
version = "0.5.2"
|
||||
@ -1289,6 +1365,16 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"windows-targets 0.52.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.8"
|
||||
@ -1399,6 +1485,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"faiss",
|
||||
"fastrand",
|
||||
"ffmpeg-the-third",
|
||||
"fnv",
|
||||
"futures-util",
|
||||
"half",
|
||||
@ -2217,6 +2304,12 @@ version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.34"
|
||||
@ -2399,6 +2492,12 @@ dependencies = [
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
||||
|
||||
[[package]]
|
||||
name = "signal-hook-registry"
|
||||
version = "1.4.2"
|
||||
|
||||
@ -39,10 +39,15 @@ url = "2"
|
||||
fastrand = "2"
|
||||
mimalloc = "0.1"
|
||||
sonic-rs = "0.3"
|
||||
ffmpeg-the-third = "2.0"
|
||||
|
||||
[patch.crates-io]
|
||||
image = { git = "https://github.com/fintelia/image/", branch = "upgrade-zune-jpeg" }
|
||||
|
||||
[[bin]]
|
||||
name = "reddit-dump"
|
||||
path = "src/reddit_dump.rs"
|
||||
path = "src/reddit_dump.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "video-reader"
|
||||
path = "src/video_reader.rs"
|
||||
@ -6,7 +6,7 @@ use anyhow::{Result, Context};
|
||||
use axum::body::Body;
|
||||
use axum::response::Response;
|
||||
use axum::{
|
||||
extract::Json,
|
||||
extract::{Json, DefaultBodyLimit},
|
||||
response::IntoResponse,
|
||||
routing::{get, post},
|
||||
Router,
|
||||
@ -860,7 +860,7 @@ async fn main() -> Result<()> {
|
||||
let client = client.clone();
|
||||
QUERIES_COUNTER.inc();
|
||||
handle_request(config, client.clone(), &index, req).await.map_err(|e| format!("{:?}", e))
|
||||
}))
|
||||
}).layer(DefaultBodyLimit::max(2<<24)))
|
||||
.route("/", get(|_req: ()| async move {
|
||||
Json(FrontendInit {
|
||||
n_total: index_.read().await.vectors.ntotal(),
|
||||
|
||||
71
src/video_reader.rs
Normal file
71
src/video_reader.rs
Normal file
@ -0,0 +1,71 @@
|
||||
extern crate ffmpeg_the_third as ffmpeg;
|
||||
use anyhow::{Result, Context};
|
||||
use image::RgbImage;
|
||||
use std::env;
|
||||
use ffmpeg::{codec, filter, format::{self, Pixel}, media::Type, software::scaling, util::frame::video::Video};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let mut ictx = format::input(&env::args().nth(1).unwrap()).context("parsing video")?;
|
||||
let video = ictx.streams().best(Type::Video).context("no video stream")?;
|
||||
let video_index = video.index();
|
||||
|
||||
let context_decoder = codec::Context::from_parameters(video.parameters()).context("initializing decoder context")?;
|
||||
let mut decoder = context_decoder.decoder().video().context("initializing decoder")?;
|
||||
|
||||
let mut graph = filter::Graph::new();
|
||||
let afr = video.avg_frame_rate();
|
||||
let afr = (((afr.0 as f32) / (afr.1 as f32)).round() as i64).max(1);
|
||||
// passing in the actual timebase breaks something, and the thumbnail filter should not need it
|
||||
graph.add(&filter::find("buffer").unwrap(), "in",
|
||||
&format!("video_size={}x{}:pix_fmt={}:time_base={}/{}:pixel_aspect={}/{}", decoder.width(), decoder.height(), decoder.format().descriptor().unwrap().name(), video.time_base().0, video.time_base().1, decoder.aspect_ratio().0, decoder.aspect_ratio().1))?;
|
||||
graph.add(&filter::find("buffersink").unwrap(), "out", "")?;
|
||||
graph.output("in", 0)?.input("out", 0)?.parse(&format!("[in] thumbnail=n={} [thumbs]; [thumbs] select='gt(scene,0.05)+eq(n,0)' [out]", afr)).context("filtergraph parse failed")?;
|
||||
let mut out = graph.get("out").unwrap();
|
||||
out.set_pixel_format(decoder.format());
|
||||
|
||||
graph.validate().context("filtergraph build failed")?;
|
||||
|
||||
let mut scaler = scaling::Context::get(
|
||||
decoder.format(),
|
||||
decoder.width(),
|
||||
decoder.height(),
|
||||
Pixel::RGB24,
|
||||
384,
|
||||
384,
|
||||
scaling::Flags::LANCZOS,
|
||||
)?;
|
||||
|
||||
let mut count = 0;
|
||||
let mut receive_and_process_decoded_frames = |decoder: &mut ffmpeg::decoder::Video, filter_graph: &mut filter::Graph| -> Result<()> {
|
||||
let mut decoded = Video::empty();
|
||||
let mut filtered = Video::empty();
|
||||
let mut rgb_frame = Video::empty();
|
||||
loop {
|
||||
if !decoder.receive_frame(&mut decoded).is_ok() { break }
|
||||
|
||||
let mut in_ctx = filter_graph.get("in").unwrap();
|
||||
let mut src = in_ctx.source();
|
||||
src.add(&decoded).context("add frame")?;
|
||||
|
||||
while filter_graph.get("out").unwrap().sink().frame(&mut filtered).is_ok() {
|
||||
scaler.run(&filtered, &mut rgb_frame).context("scaler")?;
|
||||
println!("frame gotten {}x{} {:?} {}", rgb_frame.width(), rgb_frame.height(), rgb_frame.data(0).len(), count);
|
||||
let image = RgbImage::from_vec(rgb_frame.width(), rgb_frame.height(), rgb_frame.data(0).to_vec()).unwrap(); // unfortunately, we have to copy
|
||||
image.save(format!("/tmp/output-{}.png", count))?;
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
for (stream, packet) in ictx.packets().filter_map(Result::ok) {
|
||||
if stream.index() == video_index {
|
||||
decoder.send_packet(&packet).context("decoder")?;
|
||||
receive_and_process_decoded_frames(&mut decoder, &mut graph).context("processing")?;
|
||||
}
|
||||
}
|
||||
decoder.send_eof()?;
|
||||
receive_and_process_decoded_frames(&mut decoder, &mut graph)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user