updated dependencies application wide

Cargo.toml

@@ -5,18 +5,24 @@ members = [
   "api-router",
   "html-router",
   "ingestion-pipeline",
-  "composite-retrieval"
+  "composite-retrieval",
-, "json-stream-parser"]
+  "json-stream-parser"
+]
 resolver = "2"
 
 [workspace.dependencies]
-tokio = { version = "1.40.0", features = ["full"] }
+tokio = { version = "1", features = ["full"] }
-serde = { version = "1.0.210", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
-axum = { version = "0.7.5", features = ["multipart", "macros"] }
+axum = { version = "0.8", features = ["multipart", "macros"] }
 serde_json = "1.0.128"
 thiserror = "1.0.63"
 anyhow = "1.0.94"
 tracing = "0.1.40"
-surrealdb = { version = "2.0.4", features = ["kv-mem"] }
+surrealdb = { version = "2", features = ["kv-mem"] }
 futures = "0.3.31"
 async-openai = "0.24.1"
+axum_session = "0.16"
+axum_session_auth = "0.16"
+axum_session_surreal = "0.4"
+axum_typed_multipart = "0.16"
+tempfile = "3.12.0"

api-router/Cargo.toml

@@ -4,16 +4,14 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-# Workspace dependencies
 tokio = { workspace = true }
 serde = { workspace = true }
 axum = { workspace = true }
 tracing = { workspace = true }
 anyhow = { workspace = true }
 thiserror = { workspace = true }
-
+tempfile = { workspace = true }
-tempfile = "3.12.0"
+futures = { workspace = true }
-futures = "0.3.31"
+axum_typed_multipart = { workspace = true} 
-axum_typed_multipart = "0.12.1"
 
 common = { path = "../common" }

common/Cargo.toml

@@ -15,11 +15,13 @@ serde_json = { workspace = true }
 surrealdb = { workspace = true, features = ["kv-mem"] }
 async-openai = { workspace = true }
 futures = { workspace = true }
+tempfile = { workspace = true }
 
-axum_session = "0.14.4"
+async-trait = "0.1.88"
-axum_session_auth = "0.14.1"
+axum_session = { workspace = true }
-axum_session_surreal = "0.2.1"
+axum_session_auth = { workspace = true }
-axum_typed_multipart = "0.12.1"
+axum_session_surreal =  { workspace = true}
+axum_typed_multipart = { workspace = true} 
 chrono = { version = "0.4.39", features = ["serde"] }
 chrono-tz = "0.10.1"
 config = "0.15.4"
@@ -27,7 +29,6 @@ mime = "0.3.17"
 mime_guess = "2.0.5"
 reqwest = {version = "0.12.12", features = ["charset", "json"]}
 sha2 = "0.10.8"
-tempfile = "3.12.0"
 url = { version = "2.5.2", features = ["serde"] }
 uuid = { version = "1.10.0", features = ["v4", "serde"] }
 

common/src/storage/types/analytics.rs

@@ -1,5 +1,4 @@
 use crate::storage::types::{file_info::deserialize_flexible_id, user::User, StoredObject};
-use axum::async_trait;
 use serde::{Deserialize, Serialize};
 
 use crate::{error::AppError, storage::db::SurrealDbClient};
@@ -12,7 +11,6 @@ pub struct Analytics {
     pub visitors: i64,
 }
 
-#[async_trait]
 impl StoredObject for Analytics {
     fn table_name() -> &'static str {
         "analytics"

common/src/storage/types/mod.rs

@@ -1,4 +1,3 @@
-use axum::async_trait;
 use serde::{Deserialize, Serialize};
 pub mod analytics;
 pub mod conversation;
@@ -14,7 +13,6 @@ pub mod text_chunk;
 pub mod text_content;
 pub mod user;
 
-#[async_trait]
 pub trait StoredObject: Serialize + for<'de> Deserialize<'de> {
     fn table_name() -> &'static str;
     fn get_id(&self) -> &str;
@@ -23,7 +21,6 @@ pub trait StoredObject: Serialize + for<'de> Deserialize<'de> {
 #[macro_export]
 macro_rules! stored_object {
     ($name:ident, $table:expr, {$($(#[$attr:meta])* $field:ident: $ty:ty),*}) => {
-        use axum::async_trait;
         use serde::{Deserialize, Deserializer, Serialize};
         use surrealdb::sql::Thing;
         use $crate::storage::types::StoredObject;
@@ -98,7 +95,6 @@ macro_rules! stored_object {
             $(pub $field: $ty),*
         }
 
-        #[async_trait]
         impl StoredObject for $name {
             fn table_name() -> &'static str {
                 $table

common/src/storage/types/system_settings.rs

@@ -1,5 +1,4 @@
 use crate::storage::types::file_info::deserialize_flexible_id;
-use axum::async_trait;
 use serde::{Deserialize, Serialize};
 use uuid::Uuid;
 
@@ -17,7 +16,6 @@ pub struct SystemSettings {
     pub ingestion_system_prompt: String,
 }
 
-#[async_trait]
 impl StoredObject for SystemSettings {
     fn table_name() -> &'static str {
         "system_settings"

common/src/storage/types/user.rs

@@ -1,4 +1,5 @@
 use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
+use async_trait::async_trait;
 use axum_session_auth::Authentication;
 use surrealdb::{engine::any::Any, Surreal};
 use uuid::Uuid;

devenv.lock

@@ -3,10 +3,10 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1735241861,
+        "lastModified": 1745333575,
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "991abff153b995192bf36655394246fc97ba8627",
+        "rev": "cd7456e483ca32b22b84a50015666b44217bd64f",
         "type": "github"
       },
       "original": {
@@ -31,10 +31,31 @@
         "type": "github"
       }
     },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1742649964,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "dcf5072734cb576d2b0c59b2ac44f5050b5eac82",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
-          "pre-commit-hooks",
+          "git-hooks",
           "nixpkgs"
         ]
       },
@@ -53,61 +74,27 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1733477122,
+        "lastModified": 1744868846,
-        "owner": "cachix",
+        "owner": "nixos",
-        "repo": "devenv-nixpkgs",
-        "rev": "7bd9e84d0452f6d2e63b6e6da29fe73fac951857",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "ref": "rolling",
-        "repo": "devenv-nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-stable": {
-      "locked": {
-        "lastModified": 1735286948,
-        "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "31ac92f9628682b294026f0860e14587a09ffb4b",
+        "rev": "ebe4301cbd8f81c4f8d3244b3632338bbeb6d49c",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
+        "owner": "nixos",
-        "ref": "nixos-24.05",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": "nixpkgs-stable"
-      },
-      "locked": {
-        "lastModified": 1734797603,
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "f0f0dc4920a903c3e08f5bdb9246bb572fcae498",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
         "devenv": "devenv",
+        "git-hooks": "git-hooks",
         "nixpkgs": "nixpkgs",
-        "pre-commit-hooks": "pre-commit-hooks"
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
       }
     }
   },

devenv.nix

@@ -17,7 +17,11 @@
   ];
 
   # https://devenv.sh/languages/
-  languages.rust.enable = true;
+  languages.rust = {
+    enable = true;
+    components = ["rustc" "clippy" "rustfmt" "cargo" "rust-analyzer"];
+    mold.enable = true;
+  };
 
   # https://devenv.sh/services/
   # services.postgres.enable = true;

devenv.yaml

@@ -1,7 +1,7 @@
 # yaml-language-server: $schema=https://devenv.sh/devenv.schema.json
 inputs:
   nixpkgs:
-    url: github:cachix/devenv-nixpkgs/rolling
+    url: github:nixos/nixpkgs/nixpkgs-unstable
 
 # If you're using non-OSS software, you can set allowUnfree to true.
 allowUnfree: true

html-router/Cargo.toml

@@ -12,21 +12,21 @@ tracing = { workspace = true }
 serde_json = { workspace = true }
 async-openai = { workspace = true }
 thiserror = { workspace = true }
+axum_session = { workspace = true }
+axum_session_auth = { workspace = true }
+axum_session_surreal =  { workspace = true}
+axum_typed_multipart = { workspace = true} 
+tempfile = { workspace = true }
+futures = { workspace = true }
+surrealdb = { workspace = true }
 
-axum-htmx = "0.6.0"
+axum-htmx = "0.7.0"
-axum_session = "0.14.4"
-axum_session_auth = "0.14.1"
-axum_session_surreal = "0.2.1"
-axum_typed_multipart = "0.12.1"
-futures = "0.3.31"
-tempfile = "3.12.0"
 async-stream = "0.3.6"
 minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
 minijinja-autoreload = "2.5.0"
 minijinja-embed = { version = "2.8.0" }
 minijinja-contrib = { version = "2.6.0", features = ["datetime", "timezone"] }
 plotly = "0.12.1"
-surrealdb = "2.0.4"
 tower-http = { version = "0.6.2", features = ["fs"] }
 chrono-tz = "0.10.1"
 tower-serve-static = "0.1.1"

html-router/src/middlewares/auth_middleware.rs

@@ -1,5 +1,4 @@
 use axum::{
-    async_trait,
     extract::{FromRequestParts, Request},
     http::request::Parts,
     middleware::Next,
@@ -15,7 +14,6 @@ use super::response_middleware::TemplateResponse;
 pub struct RequireUser(pub User);
 
 // Implement FromRequestParts for RequireUser
-#[async_trait]
 impl<S> FromRequestParts<S> for RequireUser
 where
     S: Send + Sync,

html-router/src/middlewares/response_middleware.rs

@@ -101,8 +101,8 @@ impl IntoResponse for TemplateResponse {
 pub async fn with_template_response<S>(
     State(state): State<S>,
     HxRequest(is_htmx): HxRequest,
-    response: Response,
+    response: Response<axum::body::Body>,
-) -> Response
+) -> Response<axum::body::Body>
 where
     S: ProvidesTemplateEngine + Clone + Send + Sync + 'static,
 {

html-router/src/routes/chat/mod.rs

@@ -25,17 +25,17 @@ where
     Router::new()
         .route("/chat", get(show_chat_base).post(new_chat_user_message))
         .route(
-            "/chat/:id",
+            "/chat/{id}",
             get(show_existing_chat)
                 .post(new_user_message)
                 .delete(delete_conversation),
         )
         .route(
-            "/chat/:id/title",
+            "/chat/{id}/title",
             get(show_conversation_editing_title).patch(patch_conversation_title),
         )
         .route("/chat/sidebar", get(reload_sidebar))
         .route("/initialized-chat", post(show_initialized_chat))
         .route("/chat/response-stream", get(get_response_stream))
-        .route("/chat/reference/:id", get(show_reference_tooltip))
+        .route("/chat/reference/{id}", get(show_reference_tooltip))
 }

html-router/src/routes/content/mod.rs

@@ -15,7 +15,7 @@ where
     Router::new()
         .route("/content", get(show_content_page))
         .route(
-            "/content/:id",
+            "/content/{id}",
             get(show_text_content_edit_form)
                 .patch(patch_text_content)
                 .delete(delete_text_content),

html-router/src/routes/index/mod.rs

@@ -23,7 +23,7 @@ where
     HtmlState: FromRef<S>,
 {
     Router::new()
-        .route("/jobs/:job_id", delete(delete_job))
+        .route("/jobs/{job_id}", delete(delete_job))
         .route("/active-jobs", get(show_active_jobs))
-        .route("/text-content/:id", delete(delete_text_content))
+        .route("/text-content/{id}", delete(delete_text_content))
 }

html-router/src/routes/knowledge/handlers.rs

@@ -50,9 +50,9 @@ pub struct KnowledgeBaseData {
 pub async fn show_knowledge_page(
     State(state): State<HtmlState>,
     RequireUser(user): RequireUser,
-    Query(mut params): Query<FilterParams>,
     HxRequest(is_htmx): HxRequest,
     HxBoosted(is_boosted): HxBoosted,
+    Query(mut params): Query<FilterParams>,
 ) -> Result<impl IntoResponse, HtmlError> {
     // Normalize filters
     params.entity_type = params.entity_type.take().filter(|s| !s.trim().is_empty());

html-router/src/routes/knowledge/mod.rs

@@ -20,14 +20,14 @@ where
     Router::new()
         .route("/knowledge", get(show_knowledge_page))
         .route(
-            "/knowledge-entity/:id",
+            "/knowledge-entity/{id}",
             get(show_edit_knowledge_entity_form)
                 .delete(delete_knowledge_entity)
                 .patch(patch_knowledge_entity),
         )
         .route("/knowledge-relationship", post(save_knowledge_relationship))
         .route(
-            "/knowledge-relationship/:id",
+            "/knowledge-relationship/{id}",
             delete(delete_knowledge_relationship),
         )
 }

ingestion-pipeline/Cargo.toml

@@ -20,6 +20,7 @@ scraper = "0.22.0"
 chrono = { version = "0.4.39", features = ["serde"] }
 text-splitter = "0.18.1"
 uuid = { version = "1.10.0", features = ["v4", "serde"] }
+dom_smoothie = "0.10.0"
 
 common = { path = "../common" }
 composite-retrieval = { path = "../composite-retrieval" }

ingestion-pipeline/src/types/mod.rs

@@ -14,6 +14,7 @@ use common::{
         text_content::TextContent,
     },
 };
+use dom_smoothie::TextMode;
 use reqwest;
 use scraper::{Html, Selector};
 use std::fmt::Write;
@@ -125,6 +126,12 @@ async fn fetch_text_from_url(
         .replace("  ", " ");
 
     process_web_content(content, openai_client, db_client).await
+
+    // let config = dom_smoothie::Config {
+    //     text_mode: TextMode::Markdown,
+    //     ..Default::default()
+    // };
+    // panic!("YOU SHALL NOT PASS");
 }
 
 pub async fn process_web_content(

main/src/server.rs

@@ -48,7 +48,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create Axum router
     let app = Router::new()
         .nest("/api/v1", api_routes_v1(&api_state))
-        .nest("/", html_routes(&html_state))
+        .merge(html_routes(&html_state))
         .with_state(AppState {
             api_state,
             html_state,

todo.md

@@ -1,3 +1,8 @@
+\[\] change to smoothie dom
+\[\] store page title
+\[\] rename ingestion instructions to context
+\[\] page screenshot?
+\[\] full text search
 \[\] archive ingressed webpage
 \[\] three js graph explorer
 \[\] three js vector explorer