mirrors
/
tagwiki
mirror of https://github.com/dpc/tagwiki
1
0
Fork 0
Code Issues Releases Wiki Activity

Something is working

This commit is contained in:
Dawid Ciężarkiewicz 2020-05-09 01:06:36 -07:00
parent cb3ade0cdb
commit e556d733b1
7 changed files with 282 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@ -24,6 +24,17 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
[[package]]
name = "async-trait"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da71fef07bc806586090247e971229289f64c210a278ee5ae419314eb386b31d"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "atty"
version = "0.2.14"
@ -1167,6 +1178,7 @@ name = "tagwiki"
version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
"blake2",
"digest",
"env_logger",

1
Cargo.toml
View File

@ -29,3 +29,4 @@ blake2 = "*"
digest = "*"
hex = "*"
walkdir = "*"
async-trait = "0.1.30"

96
src/index.rs
View File

@ -1,21 +1,23 @@
use crate::page;
use crate::page::{Id, Tag};
use crate::page::{Id, Tag, TagRef};
use anyhow::Result;
use async_trait::async_trait;
use log::info;
use std::collections::{HashMap, HashSet};
#[derive(Default)]
struct Index<T> {
pub struct Index<T> {
// tag -> page_ids
page_ids_by_tag: HashMap<String, HashSet<Id>>,
tags_by_page_id: HashMap<Id, Vec<Tag>>,
inner: T,
store: T,
}
#[derive(Default)]
struct FindResults {
matching_pages: Vec<Id>,
matching_tags: Vec<page::Tag>,
#[derive(Default, Debug, Clone)]
pub struct FindResults {
pub matching_pages: Vec<Id>,
pub matching_tags: Vec<page::Tag>,
}
impl FindResults {
@ -24,19 +26,49 @@ impl FindResults {
}
}
impl<T> Index<T>
where
T: page::StoreMut,
{
pub async fn new(store: T) -> Result<Self> {
let mut s = Index {
page_ids_by_tag: Default::default(),
tags_by_page_id: Default::default(),
store,
};
s.index_inner().await?;
Ok(s)
}
async fn index_inner(&mut self) -> Result<()> {
let mut count = 0;
let ids = self.store.iter().await?.collect::<Vec<page::Id>>();
for id in ids {
count += 1;
let page = self.store.get(id).await?;
self.add_data_for_page(&page);
}
info!("Indexed {} pages", count);
Ok(())
}
}
impl<T> Index<T> {
fn find(&self, tags: &[&Tag]) -> FindResults {
let mut matching_pages = vec![];
let mut matching_tags = vec![];
pub fn find(&self, tags: &[TagRef]) -> FindResults {
let mut matching_pages: Vec<String> = vec![];
let mut matching_tags: Vec<String> = vec![];
for tag in tags {
if matching_tags.is_empty() {
if let Some(ids) = self.page_ids_by_tag.get(tag.as_str()) {
if let Some(ids) = dbg!(&self.page_ids_by_tag).get(*tag) {
matching_pages = ids.iter().map(|id| id.to_owned()).collect();
matching_tags.push(tag.to_string())
} else {
return FindResults::empty();
}
} else {
if let Some(ids) = self.page_ids_by_tag.get(tag.as_str()) {
if let Some(ids) = self.page_ids_by_tag.get(*tag) {
let new_matching_pages: Vec<_> = matching_pages
.iter()
.filter(|id| ids.contains(id.as_str()))
@ -65,6 +97,17 @@ impl<T> Index<T> {
}
}
fn add_data_for_page(&mut self, page: &page::Parsed) {
for tag in dbg!(&page.tags) {
self.page_ids_by_tag
.entry(tag.clone())
.or_default()
.insert(page.headers.id.clone());
self.tags_by_page_id
.insert(page.headers.id.clone(), page.tags.clone());
}
}
fn clean_data_for_page(&mut self, id: Id) {
for tag in self
.tags_by_page_id
@ -80,37 +123,34 @@ impl<T> Index<T> {
}
}
#[async_trait]
impl<T> page::StoreMut for Index<T>
where
T: page::StoreMut,
T: page::StoreMut + Send + Sync,
{
fn get(&mut self, id: Id) -> Result<page::Parsed> {
self.inner.get(id)
async fn get(&self, id: Id) -> Result<page::Parsed> {
self.store.get(id).await
}
fn put(&mut self, page: &page::Parsed) -> Result<()> {
self.inner.put( page)?;
async fn put(&mut self, page: &page::Parsed) -> Result<()> {
self.store.put(page).await?;
if let Some(_tags) = self.tags_by_page_id.get(&page.headers.id) {
self.clean_data_for_page(page.headers.id.clone());
}
for tag in &page.tags {
self.page_ids_by_tag
.get_mut(tag)
.map(|set| set.insert(page.headers.id.clone()));
self.tags_by_page_id
.insert(page.headers.id.clone(), page.tags.clone());
}
self.add_data_for_page(page);
Ok(())
}
fn delete(&mut self, id: Id) -> Result<()> {
self.inner.delete(id.clone())?;
async fn delete(&mut self, id: Id) -> Result<()> {
self.store.delete(id.clone()).await?;
self.clean_data_for_page(id);
Ok(())
}
fn iter<'s>(&'s mut self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
self.inner.iter()
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
self.store.iter().await
}
}

87
src/main.rs
View File

@ -2,9 +2,12 @@
use anyhow::Result;
use log::info;
use std::sync::Arc;
use structopt::StructOpt;
use warp::{path::FullPath, Filter};
use page::StoreMut;
/// Command line options
mod cli;
/// Page
@ -15,22 +18,59 @@ mod index;
/// Utils
mod util;
async fn handler(path: FullPath) -> Result<String, std::convert::Infallible> {
#[derive(Debug)]
struct RejectAnyhow(anyhow::Error);
impl warp::reject::Reject for RejectAnyhow {}
struct State {
page_store:
Arc<tokio::sync::RwLock<index::Index<Box<dyn page::store::StoreMut + Sync + Send>>>>,
}
fn with_state(
state: Arc<State>,
) -> impl Filter<Extract = (Arc<State>,), Error = std::convert::Infallible> + Clone {
warp::any().map(move || state.clone())
}
async fn handler(
state: Arc<State>,
path: FullPath,
) -> std::result::Result<Box<dyn warp::Reply>, warp::Rejection> {
let tags: Vec<_> = path
.as_str()
.split('/')
.map(|t| t.trim())
.filter(|t| t != &"")
.collect();
Ok(format!("Path: {:?}", tags))
let read = state.page_store.read().await;
let results = read.find(tags.as_slice());
if results.matching_pages.len() == 1 {
let page = read
.get(results.matching_pages[0].clone())
.await
.map_err(|e| warp::reject::custom(RejectAnyhow(e)))?;
Ok(Box::new(warp::reply::html(page.html)))
} else {
Ok(Box::new(format!("Results: {:?}", results)))
}
}
fn start(opts: &cli::Opts) -> Result<()> {
let handler = warp::path::full().and_then(handler);
let serve = warp::serve(handler).run(([127, 0, 0, 1], opts.port));
async fn start(opts: &cli::Opts) -> Result<()> {
let state = Arc::new(State {
page_store: Arc::new(tokio::sync::RwLock::new(
index::Index::new(Box::new(page::store::FsStore::new(opts.path.clone())?)
as Box<dyn page::store::StoreMut + Send + Sync>)
.await?,
)),
});
let handler = warp::any()
.and(with_state(state))
.and(warp::path::full())
.and_then(handler);
info!("Listening on port {}", opts.port);
tokio::runtime::Runtime::new().unwrap().block_on(serve);
let _serve = warp::serve(handler).run(([127, 0, 0, 1], opts.port)).await;
Ok(())
}
@ -39,7 +79,38 @@ fn main() -> Result<()> {
env_logger::init();
let opts = cli::Opts::from_args();
start(&opts)?;
tokio::runtime::Runtime::new()
.unwrap()
.block_on(start(&opts));
Ok(())
}
/*
async fn handle_rejection(
err: warp::Rejection,
) -> Result<impl warp::Reply, std::convert::Infallible> {
use warp::http::StatusCode;
let code;
let message;
if err.is_not_found() {
code = StatusCode::NOT_FOUND;
message = "NOT_FOUND";
} else if let Some(DivideByZero) = err.find() {
code = StatusCode::BAD_REQUEST;
message = "DIVIDE_BY_ZERO";
} else if let Some(_) = err.find::<warp::reject::MethodNotAllowed>() {
// We can handle a specific error, here METHOD_NOT_ALLOWED,
// and render it however we want
code = StatusCode::METHOD_NOT_ALLOWED;
message = "METHOD_NOT_ALLOWED";
} else {
// We should have expected this... Just log and say its a 500
eprintln!("unhandled rejection: {:?}", err);
code = StatusCode::INTERNAL_SERVER_ERROR;
message = "UNHANDLED_REJECTION";
}
Ok(warp::reply::with_status(message, code))
}*/

3
src/page.rs
View File

@ -1,4 +1,4 @@
mod store;
pub mod store;
use lazy_static::lazy_static;
pub use store::{InMemoryStore, Store, StoreMut};
@ -8,6 +8,7 @@ use digest::Digest;
pub type Id = String;
pub type Tag = String;
pub type TagRef<'a> = &'a str;
const TAGWIKI_PAGE_ID_KEY: &str = "tagwiki-page-id";

126
src/page/store.rs
View File

@ -1,46 +1,125 @@
use crate::page::{self, Id};
use anyhow::{format_err, Result};
use async_trait::async_trait;
use std::collections::HashMap;
// use std::sync::{Arc, Mutex};
use std::sync;
pub mod fs;
pub use fs::FsStore;
#[async_trait]
pub trait Store {
fn get(&self, id: Id) -> Result<page::Parsed>;
fn put(&self, page: &page::Parsed) -> Result<()>;
fn delete(&self, id: Id) -> Result<()>;
fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>>;
async fn get(&self, id: Id) -> Result<page::Parsed>;
async fn put(&self, page: &page::Parsed) -> Result<()>;
async fn delete(&self, id: Id) -> Result<()>;
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>>;
}
#[async_trait]
pub trait StoreMut {
fn get(&mut self, id: Id) -> Result<page::Parsed>;
fn put(&mut self, page: &page::Parsed) -> Result<()>;
fn delete(&mut self, id: Id) -> Result<()>;
fn iter<'s>(&'s mut self) -> Result<Box<dyn Iterator<Item = Id> + 's>>;
async fn get(&self, id: Id) -> Result<page::Parsed>;
async fn put(&mut self, page: &page::Parsed) -> Result<()>;
async fn delete(&mut self, id: Id) -> Result<()>;
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>>;
}
#[async_trait]
impl<T> StoreMut for T
where
T: Store,
T: Store + Send + Sync,
{
fn get(&mut self, id: Id) -> Result<page::Parsed> {
Store::get(self, id)
async fn get(&self, id: Id) -> Result<page::Parsed> {
Store::get(self, id).await
}
fn put(&mut self, page: &page::Parsed) -> Result<()> {
Store::put(self, page)
async fn put(&mut self, page: &page::Parsed) -> Result<()> {
Store::put(self, page).await
}
fn delete(&mut self, id: Id) -> Result<()> {
Store::delete(self, id)
async fn delete(&mut self, id: Id) -> Result<()> {
Store::delete(self, id).await
}
fn iter<'s>(&'s mut self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
Store::iter(self)
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
Store::iter(self).await
}
}
// impl Store for Arc<Mutex<InMemoryStore>> {}
#[async_trait]
impl StoreMut for Box<dyn StoreMut + Send + Sync> {
async fn get(&self, id: Id) -> Result<page::Parsed> {
(**self).get(id).await
}
async fn put(&mut self, page: &page::Parsed) -> Result<()> {
(**self).put(page).await
}
async fn delete(&mut self, id: Id) -> Result<()> {
(**self).delete(id).await
}
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
(**self).iter().await
}
} /*
impl<T> Store for sync::Arc<sync::Mutex<T>>
where
T: StoreMut,
{
fn get(&self, id: Id) -> Result<page::Parsed> {
self.lock().expect("locking").get(id)
}
fn put(&self, page: &page::Parsed) -> Result<()> {
self.lock().expect("locking").put(page)
}
fn delete(&self, id: Id) -> Result<()> {
self.lock().expect("locking").delete(id)
}
fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
Ok(Box::new(
self.lock()
.expect("locking")
.iter()?
.collect::<Vec<_>>()
.into_iter(),
))
}
}
*/
#[async_trait]
impl<T> Store for sync::Arc<tokio::sync::RwLock<T>>
where
T: StoreMut + Sync + Send,
{
async fn get(&self, id: Id) -> Result<page::Parsed> {
self.read().await.get(id).await
}
async fn put(&self, page: &page::Parsed) -> Result<()> {
self.write().await.put(page).await
}
async fn delete(&self, id: Id) -> Result<()> {
self.write().await.delete(id).await
}
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
// TODO: fix that `collect`
Ok(Box::new(
self.write()
.await
.iter()
.await?
.collect::<Vec<_>>()
.into_iter(),
))
}
} // impl Store for Arc<Mutex<InMemoryStore>> {}
#[derive(Debug, Default)]
pub struct InMemoryStore {
@ -61,8 +140,9 @@ impl InMemoryStore {
*/
}
#[async_trait]
impl StoreMut for InMemoryStore {
fn get(&mut self, id: Id) -> Result<page::Parsed> {
async fn get(&self, id: Id) -> Result<page::Parsed> {
Ok(self
.page_by_id
.get(&id)
@ -70,7 +150,7 @@ impl StoreMut for InMemoryStore {
.ok_or_else(|| format_err!("Not found"))?)
}
fn put(&mut self, page: &page::Parsed) -> Result<()> {
async fn put(&mut self, page: &page::Parsed) -> Result<()> {
*self
.page_by_id
.get_mut(&page.headers.id)
@ -79,13 +159,13 @@ impl StoreMut for InMemoryStore {
Ok(())
}
fn delete(&mut self, id: Id) -> Result<()> {
async fn delete(&mut self, id: Id) -> Result<()> {
self.page_by_id
.remove(&id)
.ok_or_else(|| format_err!("Not found"))?;
Ok(())
}
fn iter<'s>(&'s mut self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
Ok(Box::new(self.page_by_id.keys().cloned()))
}
}

26
src/page/store/fs.rs
View File

@ -1,5 +1,6 @@
use crate::page::{self, Id};
use anyhow::{format_err, Result};
use anyhow::{format_err, Context, Result};
use async_trait::async_trait;
use std::collections::HashMap;
use std::ffi::OsString;
use std::io::Read;
@ -19,7 +20,7 @@ impl FsStore {
..Self::default()
};
for entry in walkdir::WalkDir::new(&s.root_path) {
match Self::try_reading_page_from_entry(&s.root_path, entry) {
match Self::try_reading_page_from_entry_res(entry) {
Ok(Some((page, path))) => {
s.id_to_path.insert(page.headers.id.clone(), path.clone());
s.path_to_page.insert(path, page);
@ -62,11 +63,17 @@ impl FsStore {
path
}
fn try_reading_page_from_entry(
root_path: &Path,
fn try_reading_page_from_entry_res(
entry: walkdir::Result<walkdir::DirEntry>,
) -> Result<Option<(page::Parsed, PathBuf)>> {
let entry = entry?;
Self::try_reading_page_from_entry(&entry)
.with_context(|| format!("While reading path: {}", entry.path().display()))
}
fn try_reading_page_from_entry(
entry: &walkdir::DirEntry,
) -> Result<Option<(page::Parsed, PathBuf)>> {
if !entry.file_type().is_file() {
return Ok(None);
}
@ -75,7 +82,7 @@ impl FsStore {
return Ok(None);
}
let file = std::fs::File::open(PathBuf::from(root_path).join(entry.path()))?;
let file = std::fs::File::open(PathBuf::from(entry.path()))?;
let mut reader = std::io::BufReader::new(file);
let mut source = page::Source::default();
reader.read_to_string(&mut source.0)?;
@ -91,15 +98,16 @@ impl FsStore {
}
}
#[async_trait]
impl page::StoreMut for FsStore {
fn get(&mut self, id: Id) -> Result<page::Parsed> {
async fn get(&self, id: Id) -> Result<page::Parsed> {
self.id_to_path
.get(&id)
.and_then(|path| self.path_to_page.get(path).cloned())
.ok_or_else(|| format_err!("Not found"))
}
fn put(&mut self, page: &page::Parsed) -> Result<()> {
async fn put(&mut self, page: &page::Parsed) -> Result<()> {
let path = if let Some(path) = self.id_to_path.get(&page.headers.id) {
path.clone()
} else {
@ -113,7 +121,7 @@ impl page::StoreMut for FsStore {
Ok(())
}
fn delete(&mut self, id: Id) -> Result<()> {
async fn delete(&mut self, id: Id) -> Result<()> {
let path = self
.id_to_path
.get(&id)
@ -125,7 +133,7 @@ impl page::StoreMut for FsStore {
Ok(())
}
fn iter<'s>(&'s mut self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
async fn iter<'s>(&'s self) -> Result<Box<dyn Iterator<Item = Id> + 's>> {
Ok(Box::new(self.id_to_path.keys().cloned()))
}
}