refacactor: tidying up server entrypoint

src/bin/server.rs

@@ -1,60 +1,12 @@
-use axum::{
+use axum::Router;
-    extract::DefaultBodyLimit,
-    middleware::from_fn_with_state,
-    routing::{delete, get, patch, post},
-    Router,
-};
-use axum_session::{SessionConfig, SessionLayer, SessionStore};
-use axum_session_auth::{AuthConfig, AuthSessionLayer};
-use axum_session_surreal::SessionSurrealPool;
-use minijinja::{path_loader, Environment};
-use minijinja_autoreload::AutoReloader;
-use std::{path::PathBuf, sync::Arc};
-use surrealdb::{engine::any::Any, Surreal};
-use tower_http::services::ServeDir;
 use tracing::info;
 use tracing_subscriber::{fmt, prelude::*, EnvFilter};
 use zettle_db::{
-    ingress::jobqueue::JobQueue,
     server::{
-        middleware_analytics::analytics_middleware,
+        routes::{api_routes_v1, html_routes},
-        middleware_api_auth::api_auth,
-        routes::{
-            api::{
-                ingress::ingress_data,
-                ingress_task::{delete_queue_task, get_queue_tasks},
-                query::query_handler,
-                queue_length::queue_length_handler,
-            },
-            html::{
-                account::{delete_account, set_api_key, show_account_page, update_timezone},
-                admin_panel::{show_admin_panel, toggle_registration_status},
-                content::{patch_text_content, show_content_page, show_text_content_edit_form},
-                documentation::{
-                    show_documentation_index, show_get_started, show_mobile_friendly,
-                    show_privacy_policy,
-                },
-                gdpr::{accept_gdpr, deny_gdpr},
-                index::{delete_job, delete_text_content, index_handler, show_active_jobs},
-                ingress_form::{hide_ingress_form, process_ingress_form, show_ingress_form},
-                knowledge::{
-                    delete_knowledge_entity, delete_knowledge_relationship, patch_knowledge_entity,
-                    save_knowledge_relationship, show_edit_knowledge_entity_form,
-                    show_knowledge_page,
-                },
-                search_result::search_result_handler,
-                signin::{authenticate_user, show_signin_form},
-                signout::sign_out_user,
-                signup::{process_signup_and_show_verification, show_signup_form},
-            },
-        },
         AppState,
     },
-    storage::{
+    utils::config::get_config,
-        db::SurrealDbClient,
-        types::{analytics::Analytics, system_settings::SystemSettings, user::User},
-    },
-    utils::{config::get_config, mailer::Mailer},
 };
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 2)]
@@ -68,72 +20,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let config = get_config()?;
 
-    info!("{:?}", config);
+    let app_state = AppState::new(&config).await?;
 
-    let reloader = AutoReloader::new(move |notifier| {
-        let template_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("templates");
-        let mut env = Environment::new();
-        env.set_loader(path_loader(&template_path));
-
-        notifier.set_fast_reload(true);
-        notifier.watch_path(&template_path, true);
-        minijinja_contrib::add_to_environment(&mut env);
-        Ok(env)
-    });
-
-    let surreal_db_client = Arc::new(
-        SurrealDbClient::new(
-            &config.surrealdb_address,
-            &config.surrealdb_username,
-            &config.surrealdb_password,
-            &config.surrealdb_namespace,
-            &config.surrealdb_database,
-        )
-        .await?,
-    );
-
-    let openai_client = Arc::new(async_openai::Client::new());
-
-    let app_state = AppState {
-        surreal_db_client: surreal_db_client.clone(),
-        templates: Arc::new(reloader),
-        openai_client: openai_client.clone(),
-        mailer: Arc::new(Mailer::new(
-            config.smtp_username,
-            config.smtp_relayer,
-            config.smtp_password,
-        )?),
-        job_queue: Arc::new(JobQueue::new(surreal_db_client, openai_client)),
-    };
-
-    let session_config = SessionConfig::default()
-        .with_table_name("test_session_table")
-        .with_secure(true);
-    let auth_config = AuthConfig::<String>::default();
-
-    let session_store: SessionStore<SessionSurrealPool<Any>> = SessionStore::new(
-        Some(app_state.surreal_db_client.client.clone().into()),
-        session_config,
-    )
-    .await?;
-
-    app_state.surreal_db_client.build_indexes().await?;
-    setup_auth(&app_state.surreal_db_client).await?;
-    Analytics::ensure_initialized(&app_state.surreal_db_client).await?;
-    SystemSettings::ensure_initialized(&app_state.surreal_db_client).await?;
-    // app_state.surreal_db_client.drop_table::<KnowledgeEntity>().await?;
     // Create Axum router
     let app = Router::new()
         .nest("/api/v1", api_routes_v1(&app_state))
-        .nest(
+        .nest("/", html_routes(&app_state))
-            "/",
-            html_routes(
-                session_store,
-                auth_config,
-                app_state.surreal_db_client.client.clone(),
-                &app_state,
-            ),
-        )
         .with_state(app_state);
 
     info!("Listening on 0.0.0.0:3000");
@@ -142,93 +34,3 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
-
-/// Router for API functionality, version 1
-fn api_routes_v1(app_state: &AppState) -> Router<AppState> {
-    Router::new()
-        // Ingress routes
-        .route("/ingress", post(ingress_data))
-        .route("/message_count", get(queue_length_handler))
-        .route("/queue", get(get_queue_tasks))
-        .route("/queue/:delivery_tag", delete(delete_queue_task))
-        .layer(DefaultBodyLimit::max(1024 * 1024 * 1024))
-        // Query routes
-        .route("/query", post(query_handler))
-        .route_layer(from_fn_with_state(app_state.clone(), api_auth))
-}
-
-/// Router for HTML endpoints
-fn html_routes(
-    session_store: SessionStore<SessionSurrealPool<Any>>,
-    auth_config: AuthConfig<String>,
-    db_client: Surreal<Any>,
-    app_state: &AppState,
-) -> Router<AppState> {
-    Router::new()
-        .route("/", get(index_handler))
-        .route("/gdpr/accept", post(accept_gdpr))
-        .route("/gdpr/deny", post(deny_gdpr))
-        .route("/search", get(search_result_handler))
-        .route("/signout", get(sign_out_user))
-        .route("/signin", get(show_signin_form).post(authenticate_user))
-        .route(
-            "/ingress-form",
-            get(show_ingress_form).post(process_ingress_form),
-        )
-        .route("/hide-ingress-form", get(hide_ingress_form))
-        .route("/text-content/:id", delete(delete_text_content))
-        .route("/jobs/:job_id", delete(delete_job))
-        .route("/active-jobs", get(show_active_jobs))
-        .route("/content", get(show_content_page))
-        .route(
-            "/content/:id",
-            get(show_text_content_edit_form).patch(patch_text_content),
-        )
-        .route("/knowledge", get(show_knowledge_page))
-        .route(
-            "/knowledge-entity/:id",
-            get(show_edit_knowledge_entity_form)
-                .delete(delete_knowledge_entity)
-                .patch(patch_knowledge_entity),
-        )
-        .route("/knowledge-relationship", post(save_knowledge_relationship))
-        .route(
-            "/knowledge-relationship/:id",
-            delete(delete_knowledge_relationship),
-        )
-        .route("/account", get(show_account_page))
-        .route("/admin", get(show_admin_panel))
-        .route("/toggle-registrations", patch(toggle_registration_status))
-        .route("/set-api-key", post(set_api_key))
-        .route("/update-timezone", patch(update_timezone))
-        .route("/delete-account", delete(delete_account))
-        .route(
-            "/signup",
-            get(show_signup_form).post(process_signup_and_show_verification),
-        )
-        .route("/documentation", get(show_documentation_index))
-        .route("/documentation/privacy-policy", get(show_privacy_policy))
-        .route("/documentation/get-started", get(show_get_started))
-        .route("/documentation/mobile-friendly", get(show_mobile_friendly))
-        .nest_service("/assets", ServeDir::new("assets/"))
-        .layer(from_fn_with_state(app_state.clone(), analytics_middleware))
-        .layer(
-            AuthSessionLayer::<User, String, SessionSurrealPool<Any>, Surreal<Any>>::new(Some(
-                db_client,
-            ))
-            .with_config(auth_config),
-        )
-        .layer(SessionLayer::new(session_store))
-}
-
-async fn setup_auth(db: &SurrealDbClient) -> Result<(), Box<dyn std::error::Error>> {
-    db.query(
-        "DEFINE TABLE user SCHEMALESS;
-        DEFINE INDEX unique_name ON TABLE user FIELDS email UNIQUE;
-        DEFINE ACCESS account ON DATABASE TYPE RECORD
-        SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id)
-        SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );",
-    )
-    .await?;
-    Ok(())
-}

src/server/mod.rs

@@ -1,8 +1,14 @@
 use crate::ingress::jobqueue::JobQueue;
 use crate::storage::db::SurrealDbClient;
+use crate::utils::config::AppConfig;
 use crate::utils::mailer::Mailer;
+use axum_session::SessionStore;
+use axum_session_surreal::SessionSurrealPool;
+use minijinja::{path_loader, Environment};
 use minijinja_autoreload::AutoReloader;
+use std::path::PathBuf;
 use std::sync::Arc;
+use surrealdb::engine::any::Any;
 
 pub mod middleware_analytics;
 pub mod middleware_api_auth;
@@ -15,4 +21,52 @@ pub struct AppState {
     pub templates: Arc<AutoReloader>,
     pub mailer: Arc<Mailer>,
     pub job_queue: Arc<JobQueue>,
+    pub session_store: Arc<SessionStore<SessionSurrealPool<Any>>>,
+}
+
+impl AppState {
+    pub async fn new(config: &AppConfig) -> Result<Self, Box<dyn std::error::Error>> {
+        let reloader = AutoReloader::new(move |notifier| {
+            let template_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("templates");
+            let mut env = Environment::new();
+            env.set_loader(path_loader(&template_path));
+
+            notifier.set_fast_reload(true);
+            notifier.watch_path(&template_path, true);
+            minijinja_contrib::add_to_environment(&mut env);
+            Ok(env)
+        });
+
+        let surreal_db_client = Arc::new(
+            SurrealDbClient::new(
+                &config.surrealdb_address,
+                &config.surrealdb_username,
+                &config.surrealdb_password,
+                &config.surrealdb_namespace,
+                &config.surrealdb_database,
+            )
+            .await?,
+        );
+
+        surreal_db_client.ensure_initialized().await?;
+
+        let openai_client = Arc::new(async_openai::Client::new());
+
+        let session_store = Arc::new(surreal_db_client.create_session_store().await?);
+
+        let app_state = AppState {
+            surreal_db_client: surreal_db_client.clone(),
+            templates: Arc::new(reloader),
+            openai_client: openai_client.clone(),
+            mailer: Arc::new(Mailer::new(
+                &config.smtp_username,
+                &config.smtp_relayer,
+                &config.smtp_password,
+            )?),
+            job_queue: Arc::new(JobQueue::new(surreal_db_client, openai_client)),
+            session_store,
+        };
+
+        Ok(app_state)
+    }
 }

src/server/routes/html/knowledge/mod.rs

@@ -199,16 +199,19 @@ pub async fn patch_knowledge_entity(
     };
 
     // Get the existing entity and validate that the user is allowed
-    User::get_and_validate_knowledge_entity(&form.id, &user.id, &state.surreal_db_client)
+    let existing_entity =
-        .await
+        User::get_and_validate_knowledge_entity(&form.id, &user.id, &state.surreal_db_client)
-        .map_err(|e| HtmlError::new(e, state.templates.clone()))?;
+            .await
+            .map_err(|e| HtmlError::new(e, state.templates.clone()))?;
 
     // Update the entity
     KnowledgeEntity::patch(
         &form.id,
         &form.name,
         &form.description,
+        &existing_entity.entity_type,
         &state.surreal_db_client,
+        &state.openai_client,
     )
     .await
     .map_err(|e| HtmlError::new(AppError::from(e), state.templates.clone()))?;

src/server/routes/mod.rs

@@ -1,2 +1,116 @@
+use api::{
+    ingress::ingress_data,
+    ingress_task::{delete_queue_task, get_queue_tasks},
+    query::query_handler,
+    queue_length::queue_length_handler,
+};
+use axum::{
+    extract::DefaultBodyLimit,
+    middleware::from_fn_with_state,
+    routing::{delete, get, patch, post},
+    Router,
+};
+use axum_session::{SessionLayer, SessionStore};
+use axum_session_auth::{AuthConfig, AuthSessionLayer};
+use axum_session_surreal::SessionSurrealPool;
+use html::{
+    account::{delete_account, set_api_key, show_account_page, update_timezone},
+    admin_panel::{show_admin_panel, toggle_registration_status},
+    content::{patch_text_content, show_content_page, show_text_content_edit_form},
+    documentation::{
+        show_documentation_index, show_get_started, show_mobile_friendly, show_privacy_policy,
+    },
+    gdpr::{accept_gdpr, deny_gdpr},
+    index::{delete_job, delete_text_content, index_handler, show_active_jobs},
+    ingress_form::{hide_ingress_form, process_ingress_form, show_ingress_form},
+    knowledge::{
+        delete_knowledge_entity, delete_knowledge_relationship, patch_knowledge_entity,
+        save_knowledge_relationship, show_edit_knowledge_entity_form, show_knowledge_page,
+    },
+    search_result::search_result_handler,
+    signin::{authenticate_user, show_signin_form},
+    signout::sign_out_user,
+    signup::{process_signup_and_show_verification, show_signup_form},
+};
+use surrealdb::{engine::any::Any, Surreal};
+use tower_http::services::ServeDir;
+
+use crate::storage::types::user::User;
+
+use super::{middleware_analytics::analytics_middleware, middleware_api_auth::api_auth, AppState};
+
 pub mod api;
 pub mod html;
+
+/// Router for API functionality, version 1
+pub fn api_routes_v1(app_state: &AppState) -> Router<AppState> {
+    Router::new()
+        // Ingress routes
+        .route("/ingress", post(ingress_data))
+        .route("/message_count", get(queue_length_handler))
+        .route("/queue", get(get_queue_tasks))
+        .route("/queue/:delivery_tag", delete(delete_queue_task))
+        .layer(DefaultBodyLimit::max(1024 * 1024 * 1024))
+        // Query routes
+        .route("/query", post(query_handler))
+        .route_layer(from_fn_with_state(app_state.clone(), api_auth))
+}
+
+/// Router for HTML endpoints
+pub fn html_routes(app_state: &AppState) -> Router<AppState> {
+    Router::new()
+        .route("/", get(index_handler))
+        .route("/gdpr/accept", post(accept_gdpr))
+        .route("/gdpr/deny", post(deny_gdpr))
+        .route("/search", get(search_result_handler))
+        .route("/signout", get(sign_out_user))
+        .route("/signin", get(show_signin_form).post(authenticate_user))
+        .route(
+            "/ingress-form",
+            get(show_ingress_form).post(process_ingress_form),
+        )
+        .route("/hide-ingress-form", get(hide_ingress_form))
+        .route("/text-content/:id", delete(delete_text_content))
+        .route("/jobs/:job_id", delete(delete_job))
+        .route("/active-jobs", get(show_active_jobs))
+        .route("/content", get(show_content_page))
+        .route(
+            "/content/:id",
+            get(show_text_content_edit_form).patch(patch_text_content),
+        )
+        .route("/knowledge", get(show_knowledge_page))
+        .route(
+            "/knowledge-entity/:id",
+            get(show_edit_knowledge_entity_form)
+                .delete(delete_knowledge_entity)
+                .patch(patch_knowledge_entity),
+        )
+        .route("/knowledge-relationship", post(save_knowledge_relationship))
+        .route(
+            "/knowledge-relationship/:id",
+            delete(delete_knowledge_relationship),
+        )
+        .route("/account", get(show_account_page))
+        .route("/admin", get(show_admin_panel))
+        .route("/toggle-registrations", patch(toggle_registration_status))
+        .route("/set-api-key", post(set_api_key))
+        .route("/update-timezone", patch(update_timezone))
+        .route("/delete-account", delete(delete_account))
+        .route(
+            "/signup",
+            get(show_signup_form).post(process_signup_and_show_verification),
+        )
+        .route("/documentation", get(show_documentation_index))
+        .route("/documentation/privacy-policy", get(show_privacy_policy))
+        .route("/documentation/get-started", get(show_get_started))
+        .route("/documentation/mobile-friendly", get(show_mobile_friendly))
+        .nest_service("/assets", ServeDir::new("assets/"))
+        .layer(from_fn_with_state(app_state.clone(), analytics_middleware))
+        .layer(
+            AuthSessionLayer::<User, String, SessionSurrealPool<Any>, Surreal<Any>>::new(Some(
+                app_state.surreal_db_client.client.clone(),
+            ))
+            .with_config(AuthConfig::<String>::default()),
+        )
+        .layer(SessionLayer::new((*app_state.session_store).clone()))
+}

src/storage/db.rs

@@ -1,4 +1,8 @@
-use super::types::StoredObject;
+use crate::error::AppError;
+
+use super::types::{analytics::Analytics, system_settings::SystemSettings, StoredObject};
+use axum_session::{SessionConfig, SessionError, SessionStore};
+use axum_session_surreal::SessionSurrealPool;
 use std::ops::Deref;
 use surrealdb::{
     engine::any::{connect, Any},
@@ -36,6 +40,40 @@ impl SurrealDbClient {
         Ok(SurrealDbClient { client: db })
     }
 
+    pub async fn create_session_store(
+        &self,
+    ) -> Result<SessionStore<SessionSurrealPool<Any>>, SessionError> {
+        SessionStore::new(
+            Some(self.client.clone().into()),
+            SessionConfig::default()
+                .with_table_name("test_session_table")
+                .with_secure(true),
+        )
+        .await
+    }
+
+    pub async fn ensure_initialized(&self) -> Result<(), AppError> {
+        Self::build_indexes(&self).await?;
+        Self::setup_auth(&self).await?;
+
+        Analytics::ensure_initialized(self).await?;
+        SystemSettings::ensure_initialized(self).await?;
+
+        Ok(())
+    }
+
+    pub async fn setup_auth(&self) -> Result<(), Error> {
+        self.client.query(
+        "DEFINE TABLE user SCHEMALESS;
+        DEFINE INDEX unique_name ON TABLE user FIELDS email UNIQUE;
+        DEFINE ACCESS account ON DATABASE TYPE RECORD
+        SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id)
+        SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );",
+    )
+    .await?;
+        Ok(())
+    }
+
     pub async fn build_indexes(&self) -> Result<(), Error> {
         self.client.query("DEFINE INDEX idx_embedding_chunks ON text_chunk FIELDS embedding HNSW DIMENSION 1536").await?;
         self.client.query("DEFINE INDEX idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536").await?;

src/storage/types/knowledge_entity.rs

@@ -1,4 +1,11 @@
-use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
+use crate::{
+    error::AppError, storage::db::SurrealDbClient, stored_object,
+    utils::embedding::generate_embedding,
+};
+use async_openai::{
+    config::{Config, OpenAIConfig},
+    Client,
+};
 use uuid::Uuid;
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -77,21 +84,31 @@ impl KnowledgeEntity {
         id: &str,
         name: &str,
         description: &str,
+        entity_type: &KnowledgeEntityType,
         db_client: &SurrealDbClient,
+        ai_client: &Client<OpenAIConfig>,
     ) -> Result<(), AppError> {
+        let embedding_input = format!(
+            "name: {}, description: {}, type: {:?}",
+            name, description, entity_type
+        );
+        let embedding = generate_embedding(ai_client, &embedding_input).await?;
+
         db_client
             .client
             .query(
                 "UPDATE type::thing($table, $id)
                 SET name = $name,
                     description = $description,
-                    updated_at = $updated_at
+                    updated_at = $updated_at,
+                    embedding = $embedding
                 RETURN AFTER",
             )
             .bind(("table", Self::table_name()))
             .bind(("id", id.to_string()))
             .bind(("name", name.to_string()))
             .bind(("updated_at", Utc::now()))
+            .bind(("embedding", embedding))
             .bind(("description", description.to_string()))
             .await?;
 

src/utils/mailer.rs

@@ -28,11 +28,11 @@ pub enum EmailError {
 
 impl Mailer {
     pub fn new(
-        username: String,
+        username: &str,
-        relayer: String,
+        relayer: &str,
-        password: String,
+        password: &str,
     ) -> Result<Self, lettre::transport::smtp::Error> {
-        let creds = Credentials::new(username, password);
+        let creds = Credentials::new(username.to_owned(), password.to_owned());
 
         let mailer = SmtpTransport::relay(&relayer)?.credentials(creds).build();
 

templates/documentation/mobile_friendly.html

@@ -1,6 +1,6 @@
 {% extends 'documentation/base.html' %}
 {% block article %}
-<h2>Mobile Friendly Ingression: How to Submit Content from iOS to Minne</h2>
+<h1>Mobile Friendly Ingression: How to Submit Content from iOS to Minne</h1>
 <p>Minne is built with simplicity in mind. Whether you wish to save a file, capture a thought, or share a page,
   submitting content is effortless. Our server provides API access that enables users to perform actions using a
   personalized API key.</p>

templates/index/hero.html

@@ -4,7 +4,10 @@
       <!-- Hero Section -->
       <h1
         class="text-5xl sm:text-6xl py-4 pt-10 font-extrabold bg-linear-to-r from-primary to-secondary text-transparent bg-clip-text font-satoshi">
-        Simplify Your Knowledge Management
+        Your Second Brain, Built to Remember
+        <div class="text-xl font-light mt-4">
+          Minne <span class="text-base-content opacity-70">/ˈmɪnɛ/ [Swedish: memory]</span>
+        </div>
       </h1>
       <p class="text-xl ">
         Capture, connect, and retrieve your knowledge effortlessly with Minne

templates/knowledge/edit_knowledge_entity_modal.html

@@ -11,11 +11,18 @@ hx-swap="outerHTML"
 
 <div class="form-control">
   <label class="floating-label">
-    <span class="label-text">Entity Name</span>
+    <span class="label-text">Name</span>
     <input type="text" name="name" value="{{ entity.name }}" class="input input-bordered w-full">
   </label>
 </div>
 
+<div class="form-control">
+  <label class="floating-label">
+    <span class="label-text">Type</span>
+    <input type="text" name="name" value="{{ entity.entity_type}}" class="input input-bordered w-full">
+  </label>
+</div>
+
 <input type="text" name="id" value="{{ entity.id }}" class="hidden">
 
 <div class="form-control">

templates/knowledge/entity_list.html

@@ -1,4 +1,4 @@
-<div class="grid sm:grid-cols-2 md:grid-cols-3 gap-4" id="entity-list">
+<div class="grid sm:grid-cols-2 lg:grid-cols-3 gap-4" id="entity-list">
   {% for entity in entities %}
   <div class="card min-w-72 bg-base-100 shadow">
     <div class="card-body">

todo.md

@@ -1,3 +1,6 @@
+\[\] chat functionality
+\[\] filtering on categories
+\[\] link to ingressed urls or archives
 \[\] archive ingressed webpage
 \[\] configs primarily get envs
 \[\] on updates of knowledgeentity create new embeddings