test concurent get q from db/cache

Reviewed-on: #3

.drone.yml

@@ -0,0 +1,13 @@
+kind: pipeline
+type: docker
+name: default
+
+steps:
+- name: build
+  image: rust:1-alpine
+  commands:
+  - apk add --no-cache musl-dev
+  - cargo fetch
+  - cargo build --all
+  environment:
+    CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse

.gitignore

@@ -1,4 +1,4 @@
 /target
 **/*.rs.bk
-
+db.dat
 /db

Cargo.toml

@@ -1,30 +1,24 @@
 [package]
 name = "qchgk_web"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Dmitry <b4tm4n@mail.ru>"]
-edition = "2018"
+edition = "2021"
+license = "MIT"
+readme = "README.md"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 
 [dependencies]
-rand="0.7"
+rand="0.8"
-serde="1.0"
+rocket = { version = "=0.5.0-rc.3", features = ["json"] }
-serde_derive="1.0"
+rocket_dyn_templates = { version = "=0.1.0-rc.3", features = ["tera"] }
-serde_json="1.0"
+chgk_ledb_lib = {git = "https://gitea.b4tman.ru/b4tman/chgk_ledb.git", rev="699478f85e", package="chgk_ledb_lib", features=["async"]}
-ledb="0.4"
+mini-moka = "0.10.0"
-ledb-derive="0.4"
-ledb-types="0.4"
-rocket="0.4"
-
-# lmdb-zero="0.4"
-
-[dependencies.rocket_contrib]
-version = "0.4"
-default-features = false
-features = ["serve", "tera_templates"]
 
 [profile.release]
 opt-level = 3
 debug = false
 lto = true
+strip = true
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Dmitry Belyaev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,10 @@
+# qchgk_web
+
+Простой веб интерфейс к базе вопросов ЧГК
+
+## Ссылки
+
+- Источник вопросов: http://db.chgk.info
+- Копия файлов базы вопросов: https://gitlab.com/b4tman/db_chgk
+- Конвертор в JSON: https://gitea.b4tman.ru/b4tman/chgk_txt2json
+- Загрузчик в формат базы для этой штуки: https://gitea.b4tman.ru/b4tman/chgk_ledb

src/main.rs

@@ -1,187 +1,161 @@
-#![feature(proc_macro_hygiene, decl_macro)]
-
-extern crate serde;
-#[macro_use]
-extern crate serde_derive;
-extern crate ledb;
-extern crate serde_json;
-#[macro_use]
-extern crate ledb_derive;
-extern crate ledb_types;
-
-//#[macro_use]
-//extern crate tera;
-
-//use tera::Context;
-
 #[macro_use]
 extern crate rocket;
-extern crate rocket_contrib;
 
+use rocket::fs::FileServer;
+use rocket::futures::FutureExt;
 use rocket::response::Redirect;
-use rocket::{Rocket, State};
+use rocket::State;
-use rocket_contrib::serve::StaticFiles;
+use rocket_dyn_templates::tera;
-use rocket_contrib::templates::Template;
+use rocket_dyn_templates::Template;
 
 use rand::distributions::Uniform;
 use rand::Rng;
 
-use ledb::{Options, Storage};
+use std::ops::Deref;
+use std::sync::Arc;
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
+use chgk_ledb_lib::async_db;
-struct BatchInfo {
+use chgk_ledb_lib::questions::Question;
-    #[document(primary)]
+
-    #[serde(default)]
+use mini_moka::sync::Cache;
-    filename: String,
+use std::time::Duration;
-    #[serde(default)]
+
-    description: String,
+const DB_FILENAME: &str = "db.dat";
-    #[serde(default)]
+
-    author: String,
+trait ErrorEmpty {
-    #[serde(default)]
+    type Output;
-    comment: String,
+    fn err_empty(self) -> Result<Self::Output, ()>;
-    #[serde(default)]
-    url: String,
-    #[serde(default)]
-    date: String,
-    #[serde(default)]
-    processed_by: String,
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(default)]
-    copyright: String,
-    #[serde(default)]
-    theme: String,
-    #[serde(default)]
-    kind: String,
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    rating: String,
 }
 
-#[derive(Debug, Default, Clone, Serialize, Deserialize, Document)]
+impl<T, E> ErrorEmpty for Result<T, E> {
-struct Question {
+    type Output = T;
-    #[document(primary)]
+    fn err_empty(self) -> Result<Self::Output, ()> {
-    #[serde(default)]
+        self.map_err(|_| ())
-    num: u32,
+    }
-    #[document(index)]
-    id: String,
-
-    description: String,
-    answer: String,
-
-    #[serde(default)]
-    author: String,
-    #[serde(default)]
-    comment: String,
-    #[serde(default)]
-    comment1: String,
-    #[serde(default)]
-    tour: String,
-    #[serde(default)]
-    url: String,
-    #[serde(default)]
-    date: String,
-    #[serde(default)]
-    processed_by: String,
-    #[serde(default)]
-    redacted_by: String,
-    #[serde(default)]
-    copyright: String,
-    #[serde(default)]
-    theme: String,
-    #[serde(default)]
-    kind: String,
-    #[serde(default)]
-    source: String,
-    #[serde(default)]
-    rating: String,
-    #[document(nested)]
-    #[serde(default)]
-    batch_info: BatchInfo,
 }
 
+#[derive(Clone)]
+struct ArcTemplateData {
+    value: Arc<tera::Value>,
+}
+
+impl ArcTemplateData {
+    fn new(value: tera::Value) -> ArcTemplateData {
+        ArcTemplateData {
+            value: Arc::new(value),
+        }
+    }
+    fn render(&self, name: &'static str) -> Template {
+        Template::render(name, self.value.deref())
+    }
+}
+
+type TemplateCache = mini_moka::sync::Cache<usize, ArcTemplateData>;
+
+type DataBaseInner = async_db::Reader<Question>;
+type DataBase = Arc<DataBaseInner>;
 struct AppState {
-    storage: Storage,
+    db: DataBase,
-    database_distribution: Uniform<u32>,
+    database_distribution: Uniform<usize>,
 }
 
-fn get_database_distribution(storage: &Storage) -> Uniform<u32> {
+impl From<DataBaseInner> for AppState {
-    let collection = storage
+    fn from(db: DataBaseInner) -> Self {
-        .collection("questions")
+        let last_id = db.len();
-        .expect("collection \"questions\"");
+        let database_distribution = rand::distributions::Uniform::new_inclusive(1usize, last_id);
-    let last_id = collection.last_id().expect("\"questions\" last_id");
+        let db = Arc::new(db);
 
-    rand::distributions::Uniform::new_inclusive(1u32, last_id)
+        Self {
+            db,
+            database_distribution,
+        }
+    }
 }
 
-fn random_question_id(database_distribution: &Uniform<u32>) -> u32 {
+fn random_question_id(database_distribution: &Uniform<usize>) -> usize {
     let mut rng = rand::thread_rng();
     rng.sample(database_distribution)
 }
 
-fn get_question(storage: &Storage, id: u32) -> Result<Option<Question>, ()> {
+async fn get_question(db: &DataBase, id: usize) -> Result<Question, ()> {
-    if 0 == id {
+    db.get(id - 1).await.err_empty()
-        return Ok(None);
-    }
-
-    let collection = storage
-        .collection("questions")
-        .expect("collection \"questions\"");
-    let last_id = collection.last_id().expect("\"questions\" last_id");
-
-    if id > last_id {
-        Err(())
-    } else {
-        let question = collection.get::<Question>(id);
-        if question.is_err() {
-            Err(())
-        } else {
-            Ok(question.unwrap())
-        }
-    }
 }
 
-fn show_question_details(
+async fn show_question_details(
     template_name: &'static str,
     data: &AppState,
-    id: u32,
+    cache: &TemplateCache,
-) -> Result<Template, Redirect> {
+    id: usize,
-    let question = get_question(&data.storage, id);
+) -> Template {
+    let cache_cloned = cache.clone();
 
-    if question.is_ok() {
+    let from_cache = rocket::tokio::spawn(async move { cache_cloned.get(&id) }).fuse();
-        let question = question.unwrap();
+    let from_db = async move {
+        match get_question(&data.db, id).await {
+            Ok(question) => {
+                let mut context = tera::to_value(question).expect("question serialize");
+                if context.is_object() {
+                    let next_id = random_question_id(&data.database_distribution);
+                    context["next"] = tera::to_value(next_id).expect("question id serialize");
+                }
 
-        if question.is_some() {
+                let value = ArcTemplateData::new(context);
-            let question = question.unwrap();
+                let result = value.render(template_name);
-            let mut context = serde_json::to_value(question).expect("question serialize");
+                cache.insert(id, value);
-            if context.is_object() {
+
-                let next_id = random_question_id(&data.database_distribution);
+                result
-                context["next"] = serde_json::to_value(next_id).expect("question id serialize");
+            }
+            Err(_) => {
+                use std::collections::HashMap;
+                let context: HashMap<String, String> = HashMap::new();
+                Template::render("404", context)
+            }
+        }
+    }
+    .fuse();
+
+    loop {
+        rocket::tokio::select! {
+            biased;
+
+            Ok(Some(template)) = from_cache => {
+                println!("from cache");
+                break template.render(template_name)
+            },
+            template = from_db => {
+                println!("from db");
+                break template
             }
-            Ok(Template::render(template_name, &context))
-        } else {
-            Err(Redirect::permanent("/q/"))
         }
-    } else {
-        use std::collections::HashMap;
-        let context: HashMap<String, String> = HashMap::new();
-        Ok(Template::render("404", &context))
     }
 }
 
 #[get("/q/<id>")]
-fn show_question(data: State<AppState>, id: u32) -> Result<Template, Redirect> {
+async fn show_question(
-    show_question_details("question", data.inner(), id)
+    data: &State<AppState>,
+    cache: &State<TemplateCache>,
+    id: usize,
+) -> Template {
+    show_question_details("question", data.inner(), cache.inner(), id).await
 }
 
 #[get("/q/<id>/a")]
-fn show_answer(data: State<AppState>, id: u32) -> Result<Template, Redirect> {
+async fn show_answer(data: &State<AppState>, cache: &State<TemplateCache>, id: usize) -> Template {
-    show_question_details("answer", data.inner(), id)
+    show_question_details("answer", data.inner(), cache.inner(), id).await
+}
+
+#[get("/q/0")]
+fn question0() -> Redirect {
+    Redirect::to("/")
+}
+
+#[get("/q/0/a")]
+fn answer0() -> Redirect {
+    Redirect::to("/")
 }
 
 #[get("/")]
-fn index(data: State<AppState>) -> Redirect {
+fn index(data: &State<AppState>) -> Redirect {
     let id = random_question_id(&data.database_distribution);
     Redirect::temporary(format!("/q/{}", id))
 }
@@ -190,33 +164,29 @@ fn index(data: State<AppState>) -> Redirect {
 fn not_found(_req: &rocket::Request) -> Template {
     use std::collections::HashMap;
     let context: HashMap<String, String> = HashMap::new();
-    Template::render("404", &context)
+    Template::render("404", context)
 }
 
-fn rocket() -> Rocket {
+#[launch]
-    let options: Options = serde_json::from_value(serde_json::json!({
+async fn rocket() -> _ {
-    "read_only": true,
+    let state: AppState = async_db::Reader::new(DB_FILENAME)
-    "no_lock": true,
+        .await
-    }))
+        .expect("open db")
-    .unwrap();
+        .into();
+    let cache: TemplateCache = Cache::builder()
+        .time_to_idle(Duration::from_secs(15 * 60))
+        .max_capacity(300)
+        .build();
 
-    let mut state = AppState {
+    rocket::build()
-        storage: Storage::new("db", options).expect("open db"),
-        database_distribution: Uniform::new(1, 3),
-    };
-    state.database_distribution = get_database_distribution(&state.storage);
-
-    let state = state;
-
-    rocket::ignite()
         .manage(state)
-        .register(catchers![not_found])
+        .manage(cache)
-        .mount("/", routes![index, show_question, show_answer])
+        .register("/", catchers![not_found])
+        .mount(
+            "/",
+            routes![index, show_question, show_answer, question0, answer0],
+        )
         .mount("/q", routes![index])
-        .mount("/q/static", StaticFiles::from("static/"))
+        .mount("/q/static", FileServer::from("static/"))
         .attach(Template::fairing())
 }
-
-fn main() {
-    rocket().launch();
-}