diff --git a/Cargo.lock b/Cargo.lock index 4ea3d3c..3d863c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,17 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" +[[package]] +name = "async-trait" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da71fef07bc806586090247e971229289f64c210a278ee5ae419314eb386b31d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -1167,6 +1178,7 @@ name = "tagwiki" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "blake2", "digest", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index 5b97060..97a4c7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,3 +29,4 @@ blake2 = "*" digest = "*" hex = "*" walkdir = "*" +async-trait = "0.1.30" diff --git a/src/index.rs b/src/index.rs index a83f4c1..4f10c88 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,21 +1,23 @@ use crate::page; -use crate::page::{Id, Tag}; +use crate::page::{Id, Tag, TagRef}; use anyhow::Result; +use async_trait::async_trait; +use log::info; use std::collections::{HashMap, HashSet}; #[derive(Default)] -struct Index { +pub struct Index { // tag -> page_ids page_ids_by_tag: HashMap>, tags_by_page_id: HashMap>, - inner: T, + store: T, } -#[derive(Default)] -struct FindResults { - matching_pages: Vec, - matching_tags: Vec, +#[derive(Default, Debug, Clone)] +pub struct FindResults { + pub matching_pages: Vec, + pub matching_tags: Vec, } impl FindResults { @@ -24,19 +26,49 @@ impl FindResults { } } +impl Index +where + T: page::StoreMut, +{ + pub async fn new(store: T) -> Result { + let mut s = Index { + page_ids_by_tag: Default::default(), + tags_by_page_id: Default::default(), + store, + }; + + s.index_inner().await?; + + Ok(s) + } + + async fn index_inner(&mut self) -> Result<()> { + let mut count = 0; + let ids = self.store.iter().await?.collect::>(); + for id in ids { + count += 1; + let page = self.store.get(id).await?; + self.add_data_for_page(&page); + } + info!("Indexed {} pages", count); + Ok(()) + } +} + impl Index { - fn find(&self, tags: &[&Tag]) -> FindResults { - let mut matching_pages = vec![]; - let mut matching_tags = vec![]; + pub fn find(&self, tags: &[TagRef]) -> FindResults { + let mut matching_pages: Vec = vec![]; + let mut matching_tags: Vec = vec![]; for tag in tags { if matching_tags.is_empty() { - if let Some(ids) = self.page_ids_by_tag.get(tag.as_str()) { + if let Some(ids) = dbg!(&self.page_ids_by_tag).get(*tag) { matching_pages = ids.iter().map(|id| id.to_owned()).collect(); + matching_tags.push(tag.to_string()) } else { return FindResults::empty(); } } else { - if let Some(ids) = self.page_ids_by_tag.get(tag.as_str()) { + if let Some(ids) = self.page_ids_by_tag.get(*tag) { let new_matching_pages: Vec<_> = matching_pages .iter() .filter(|id| ids.contains(id.as_str())) @@ -65,6 +97,17 @@ impl Index { } } + fn add_data_for_page(&mut self, page: &page::Parsed) { + for tag in dbg!(&page.tags) { + self.page_ids_by_tag + .entry(tag.clone()) + .or_default() + .insert(page.headers.id.clone()); + self.tags_by_page_id + .insert(page.headers.id.clone(), page.tags.clone()); + } + } + fn clean_data_for_page(&mut self, id: Id) { for tag in self .tags_by_page_id @@ -80,37 +123,34 @@ impl Index { } } +#[async_trait] impl page::StoreMut for Index where - T: page::StoreMut, + T: page::StoreMut + Send + Sync, { - fn get(&mut self, id: Id) -> Result { - self.inner.get(id) + async fn get(&self, id: Id) -> Result { + self.store.get(id).await } - fn put(&mut self, page: &page::Parsed) -> Result<()> { - self.inner.put( page)?; + + async fn put(&mut self, page: &page::Parsed) -> Result<()> { + self.store.put(page).await?; if let Some(_tags) = self.tags_by_page_id.get(&page.headers.id) { self.clean_data_for_page(page.headers.id.clone()); } - for tag in &page.tags { - self.page_ids_by_tag - .get_mut(tag) - .map(|set| set.insert(page.headers.id.clone())); - self.tags_by_page_id - .insert(page.headers.id.clone(), page.tags.clone()); - } + self.add_data_for_page(page); Ok(()) } - fn delete(&mut self, id: Id) -> Result<()> { - self.inner.delete(id.clone())?; + + async fn delete(&mut self, id: Id) -> Result<()> { + self.store.delete(id.clone()).await?; self.clean_data_for_page(id); Ok(()) } - fn iter<'s>(&'s mut self) -> Result + 's>> { - self.inner.iter() + async fn iter<'s>(&'s self) -> Result + 's>> { + self.store.iter().await } } diff --git a/src/main.rs b/src/main.rs index a4a2967..99f8baf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,9 +2,12 @@ use anyhow::Result; use log::info; +use std::sync::Arc; use structopt::StructOpt; use warp::{path::FullPath, Filter}; +use page::StoreMut; + /// Command line options mod cli; /// Page @@ -15,22 +18,59 @@ mod index; /// Utils mod util; -async fn handler(path: FullPath) -> Result { +#[derive(Debug)] +struct RejectAnyhow(anyhow::Error); + +impl warp::reject::Reject for RejectAnyhow {} + +struct State { + page_store: + Arc>>>, +} + +fn with_state( + state: Arc, +) -> impl Filter,), Error = std::convert::Infallible> + Clone { + warp::any().map(move || state.clone()) +} + +async fn handler( + state: Arc, + path: FullPath, +) -> std::result::Result, warp::Rejection> { let tags: Vec<_> = path .as_str() .split('/') .map(|t| t.trim()) .filter(|t| t != &"") .collect(); - Ok(format!("Path: {:?}", tags)) + let read = state.page_store.read().await; + let results = read.find(tags.as_slice()); + if results.matching_pages.len() == 1 { + let page = read + .get(results.matching_pages[0].clone()) + .await + .map_err(|e| warp::reject::custom(RejectAnyhow(e)))?; + Ok(Box::new(warp::reply::html(page.html))) + } else { + Ok(Box::new(format!("Results: {:?}", results))) + } } -fn start(opts: &cli::Opts) -> Result<()> { - let handler = warp::path::full().and_then(handler); - let serve = warp::serve(handler).run(([127, 0, 0, 1], opts.port)); +async fn start(opts: &cli::Opts) -> Result<()> { + let state = Arc::new(State { + page_store: Arc::new(tokio::sync::RwLock::new( + index::Index::new(Box::new(page::store::FsStore::new(opts.path.clone())?) + as Box) + .await?, + )), + }); + let handler = warp::any() + .and(with_state(state)) + .and(warp::path::full()) + .and_then(handler); info!("Listening on port {}", opts.port); - - tokio::runtime::Runtime::new().unwrap().block_on(serve); + let _serve = warp::serve(handler).run(([127, 0, 0, 1], opts.port)).await; Ok(()) } @@ -39,7 +79,38 @@ fn main() -> Result<()> { env_logger::init(); let opts = cli::Opts::from_args(); - start(&opts)?; + tokio::runtime::Runtime::new() + .unwrap() + .block_on(start(&opts)); Ok(()) } + +/* +async fn handle_rejection( + err: warp::Rejection, +) -> Result { + use warp::http::StatusCode; + let code; + let message; + + if err.is_not_found() { + code = StatusCode::NOT_FOUND; + message = "NOT_FOUND"; + } else if let Some(DivideByZero) = err.find() { + code = StatusCode::BAD_REQUEST; + message = "DIVIDE_BY_ZERO"; + } else if let Some(_) = err.find::() { + // We can handle a specific error, here METHOD_NOT_ALLOWED, + // and render it however we want + code = StatusCode::METHOD_NOT_ALLOWED; + message = "METHOD_NOT_ALLOWED"; + } else { + // We should have expected this... Just log and say its a 500 + eprintln!("unhandled rejection: {:?}", err); + code = StatusCode::INTERNAL_SERVER_ERROR; + message = "UNHANDLED_REJECTION"; + } + + Ok(warp::reply::with_status(message, code)) +}*/ diff --git a/src/page.rs b/src/page.rs index a40cbbb..9390caf 100644 --- a/src/page.rs +++ b/src/page.rs @@ -1,4 +1,4 @@ -mod store; +pub mod store; use lazy_static::lazy_static; pub use store::{InMemoryStore, Store, StoreMut}; @@ -8,6 +8,7 @@ use digest::Digest; pub type Id = String; pub type Tag = String; +pub type TagRef<'a> = &'a str; const TAGWIKI_PAGE_ID_KEY: &str = "tagwiki-page-id"; diff --git a/src/page/store.rs b/src/page/store.rs index b0dade8..e59a002 100644 --- a/src/page/store.rs +++ b/src/page/store.rs @@ -1,46 +1,125 @@ use crate::page::{self, Id}; use anyhow::{format_err, Result}; +use async_trait::async_trait; use std::collections::HashMap; -// use std::sync::{Arc, Mutex}; +use std::sync; pub mod fs; +pub use fs::FsStore; +#[async_trait] pub trait Store { - fn get(&self, id: Id) -> Result; - fn put(&self, page: &page::Parsed) -> Result<()>; - fn delete(&self, id: Id) -> Result<()>; - fn iter<'s>(&'s self) -> Result + 's>>; + async fn get(&self, id: Id) -> Result; + async fn put(&self, page: &page::Parsed) -> Result<()>; + async fn delete(&self, id: Id) -> Result<()>; + async fn iter<'s>(&'s self) -> Result + 's>>; } +#[async_trait] pub trait StoreMut { - fn get(&mut self, id: Id) -> Result; - fn put(&mut self, page: &page::Parsed) -> Result<()>; - fn delete(&mut self, id: Id) -> Result<()>; - fn iter<'s>(&'s mut self) -> Result + 's>>; + async fn get(&self, id: Id) -> Result; + async fn put(&mut self, page: &page::Parsed) -> Result<()>; + async fn delete(&mut self, id: Id) -> Result<()>; + async fn iter<'s>(&'s self) -> Result + 's>>; } +#[async_trait] impl StoreMut for T where - T: Store, + T: Store + Send + Sync, { - fn get(&mut self, id: Id) -> Result { - Store::get(self, id) + async fn get(&self, id: Id) -> Result { + Store::get(self, id).await } - fn put(&mut self, page: &page::Parsed) -> Result<()> { - Store::put(self, page) + async fn put(&mut self, page: &page::Parsed) -> Result<()> { + Store::put(self, page).await } - fn delete(&mut self, id: Id) -> Result<()> { - Store::delete(self, id) + async fn delete(&mut self, id: Id) -> Result<()> { + Store::delete(self, id).await } - fn iter<'s>(&'s mut self) -> Result + 's>> { - Store::iter(self) + async fn iter<'s>(&'s self) -> Result + 's>> { + Store::iter(self).await } } -// impl Store for Arc> {} +#[async_trait] +impl StoreMut for Box { + async fn get(&self, id: Id) -> Result { + (**self).get(id).await + } + + async fn put(&mut self, page: &page::Parsed) -> Result<()> { + (**self).put(page).await + } + + async fn delete(&mut self, id: Id) -> Result<()> { + (**self).delete(id).await + } + + async fn iter<'s>(&'s self) -> Result + 's>> { + (**self).iter().await + } +} /* + impl Store for sync::Arc> + where + T: StoreMut, + { + fn get(&self, id: Id) -> Result { + self.lock().expect("locking").get(id) + } + + fn put(&self, page: &page::Parsed) -> Result<()> { + self.lock().expect("locking").put(page) + } + + fn delete(&self, id: Id) -> Result<()> { + self.lock().expect("locking").delete(id) + } + + fn iter<'s>(&'s self) -> Result + 's>> { + Ok(Box::new( + self.lock() + .expect("locking") + .iter()? + .collect::>() + .into_iter(), + )) + } + } + */ + +#[async_trait] +impl Store for sync::Arc> +where + T: StoreMut + Sync + Send, +{ + async fn get(&self, id: Id) -> Result { + self.read().await.get(id).await + } + + async fn put(&self, page: &page::Parsed) -> Result<()> { + self.write().await.put(page).await + } + + async fn delete(&self, id: Id) -> Result<()> { + self.write().await.delete(id).await + } + + async fn iter<'s>(&'s self) -> Result + 's>> { + // TODO: fix that `collect` + Ok(Box::new( + self.write() + .await + .iter() + .await? + .collect::>() + .into_iter(), + )) + } +} // impl Store for Arc> {} #[derive(Debug, Default)] pub struct InMemoryStore { @@ -61,8 +140,9 @@ impl InMemoryStore { */ } +#[async_trait] impl StoreMut for InMemoryStore { - fn get(&mut self, id: Id) -> Result { + async fn get(&self, id: Id) -> Result { Ok(self .page_by_id .get(&id) @@ -70,7 +150,7 @@ impl StoreMut for InMemoryStore { .ok_or_else(|| format_err!("Not found"))?) } - fn put(&mut self, page: &page::Parsed) -> Result<()> { + async fn put(&mut self, page: &page::Parsed) -> Result<()> { *self .page_by_id .get_mut(&page.headers.id) @@ -79,13 +159,13 @@ impl StoreMut for InMemoryStore { Ok(()) } - fn delete(&mut self, id: Id) -> Result<()> { + async fn delete(&mut self, id: Id) -> Result<()> { self.page_by_id .remove(&id) .ok_or_else(|| format_err!("Not found"))?; Ok(()) } - fn iter<'s>(&'s mut self) -> Result + 's>> { + async fn iter<'s>(&'s self) -> Result + 's>> { Ok(Box::new(self.page_by_id.keys().cloned())) } } diff --git a/src/page/store/fs.rs b/src/page/store/fs.rs index fcf2e94..d456ea1 100644 --- a/src/page/store/fs.rs +++ b/src/page/store/fs.rs @@ -1,5 +1,6 @@ use crate::page::{self, Id}; -use anyhow::{format_err, Result}; +use anyhow::{format_err, Context, Result}; +use async_trait::async_trait; use std::collections::HashMap; use std::ffi::OsString; use std::io::Read; @@ -19,7 +20,7 @@ impl FsStore { ..Self::default() }; for entry in walkdir::WalkDir::new(&s.root_path) { - match Self::try_reading_page_from_entry(&s.root_path, entry) { + match Self::try_reading_page_from_entry_res(entry) { Ok(Some((page, path))) => { s.id_to_path.insert(page.headers.id.clone(), path.clone()); s.path_to_page.insert(path, page); @@ -62,11 +63,17 @@ impl FsStore { path } - fn try_reading_page_from_entry( - root_path: &Path, + fn try_reading_page_from_entry_res( entry: walkdir::Result, ) -> Result> { let entry = entry?; + Self::try_reading_page_from_entry(&entry) + .with_context(|| format!("While reading path: {}", entry.path().display())) + } + + fn try_reading_page_from_entry( + entry: &walkdir::DirEntry, + ) -> Result> { if !entry.file_type().is_file() { return Ok(None); } @@ -75,7 +82,7 @@ impl FsStore { return Ok(None); } - let file = std::fs::File::open(PathBuf::from(root_path).join(entry.path()))?; + let file = std::fs::File::open(PathBuf::from(entry.path()))?; let mut reader = std::io::BufReader::new(file); let mut source = page::Source::default(); reader.read_to_string(&mut source.0)?; @@ -91,15 +98,16 @@ impl FsStore { } } +#[async_trait] impl page::StoreMut for FsStore { - fn get(&mut self, id: Id) -> Result { + async fn get(&self, id: Id) -> Result { self.id_to_path .get(&id) .and_then(|path| self.path_to_page.get(path).cloned()) .ok_or_else(|| format_err!("Not found")) } - fn put(&mut self, page: &page::Parsed) -> Result<()> { + async fn put(&mut self, page: &page::Parsed) -> Result<()> { let path = if let Some(path) = self.id_to_path.get(&page.headers.id) { path.clone() } else { @@ -113,7 +121,7 @@ impl page::StoreMut for FsStore { Ok(()) } - fn delete(&mut self, id: Id) -> Result<()> { + async fn delete(&mut self, id: Id) -> Result<()> { let path = self .id_to_path .get(&id) @@ -125,7 +133,7 @@ impl page::StoreMut for FsStore { Ok(()) } - fn iter<'s>(&'s mut self) -> Result + 's>> { + async fn iter<'s>(&'s self) -> Result + 's>> { Ok(Box::new(self.id_to_path.keys().cloned())) } }