fix: arc-share retrieved chunks, centralize entity embeddings, and trim hot-path clones.

html-router/src/html_state.rs

@@ -142,7 +142,9 @@ impl HtmlState {
             return;
         }
 
-        let overflow = cache.len().saturating_sub(CONVERSATION_ARCHIVE_CACHE_MAX_USERS);
+        let overflow = cache
+            .len()
+            .saturating_sub(CONVERSATION_ARCHIVE_CACHE_MAX_USERS);
         let mut by_expiry: Vec<(String, Instant)> = cache
             .iter()
             .map(|(user_id, entry)| (user_id.clone(), entry.expires_at))

html-router/src/middlewares/response_middleware.rs

@@ -183,9 +183,7 @@ fn forward_headers(from: &axum::http::HeaderMap, to: &mut axum::http::HeaderMap)
     }
 }
 
-fn context_to_map(
-    value: &Value,
-) -> Result<HashMap<String, Value>, minijinja::value::ValueKind> {
+fn context_to_map(value: &Value) -> Result<HashMap<String, Value>, minijinja::value::ValueKind> {
     match value.kind() {
         minijinja::value::ValueKind::Map => {
             let mut map = HashMap::new();

html-router/src/router_factory.rs

@@ -8,8 +8,8 @@ use surrealdb::{engine::any::Any, Surreal};
 use crate::{
     html_state::HtmlState,
     middlewares::{
-        analytics_middleware::analytics_middleware, auth_middleware::require_auth,
-        compression, response_middleware::with_template_response,
+        analytics_middleware::analytics_middleware, auth_middleware::require_auth, compression,
+        response_middleware::with_template_response,
     },
 };
 

html-router/src/routes/account/handlers.rs

@@ -164,9 +164,7 @@ pub async fn update_theme(
     ))
 }
 
-pub async fn show_change_password(
-    RequireUser(_user): RequireUser,
-) -> TemplateResult {
+pub async fn show_change_password(RequireUser(_user): RequireUser) -> TemplateResult {
     Ok(TemplateResponse::new_template(
         "auth/change_password_form.html",
         (),

html-router/src/routes/admin/handlers.rs

@@ -18,8 +18,8 @@ use common::{
     utils::{
         config::AppConfig,
         embedding::{
-            fastembed_model_dimension, is_valid_fastembed_model_code, list_fastembed_embedding_models,
-            EmbeddingBackend, FastEmbedModelOption,
+            fastembed_model_dimension, is_valid_fastembed_model_code,
+            list_fastembed_embedding_models, EmbeddingBackend, FastEmbedModelOption,
         },
     },
 };
@@ -52,7 +52,6 @@ pub enum AdminSection {
     Models,
 }
 
-
 #[derive(Deserialize)]
 pub struct AdminPanelQuery {
     section: Option<String>,
@@ -101,8 +100,9 @@ pub async fn show_admin_panel(
             (None, None, false)
         };
 
-    let effective_backend =
-        effective_embedding_backend(&settings, &state.config).as_str().to_string();
+    let effective_backend = effective_embedding_backend(&settings, &state.config)
+        .as_str()
+        .to_string();
 
     Ok(TemplateResponse::new_template(
         "admin/base.html",
@@ -187,7 +187,9 @@ struct EmbeddingSettingsPlan {
 }
 
 fn effective_embedding_backend(settings: &SystemSettings, config: &AppConfig) -> EmbeddingBackend {
-    settings.embedding_backend.unwrap_or(config.embedding_backend)
+    settings
+        .embedding_backend
+        .unwrap_or(config.embedding_backend)
 }
 
 fn is_fastembed_admin_context(settings: &SystemSettings, config: &AppConfig) -> bool {
@@ -241,11 +243,10 @@ fn plan_embedding_settings_update(
                 )));
             }
 
-            let embedding_dimensions = fastembed_model_dimension(&embedding_model)
-                .map_err(AppError::from)?;
+            let embedding_dimensions =
+                fastembed_model_dimension(&embedding_model).map_err(AppError::from)?;
             let reembedding_needed = embedding_dimensions != current.embedding_dimensions;
-            let restart_needed =
-                embedding_model != current.embedding_model || reembedding_needed;
+            let restart_needed = embedding_model != current.embedding_model || reembedding_needed;
 
             Ok(EmbeddingSettingsPlan {
                 embedding_model,
@@ -274,8 +275,7 @@ pub async fn update_model_settings(
     Form(input): Form<ModelSettingsInput>,
 ) -> TemplateResult {
     let current_settings = SystemSettings::get_current(&state.db).await?;
-    let embedding_plan =
-        plan_embedding_settings_update(&current_settings, &input, &state.config)?;
+    let embedding_plan = plan_embedding_settings_update(&current_settings, &input, &state.config)?;
 
     let new_settings = SystemSettingsPatch {
         query_model: Some(input.query_model),
@@ -309,10 +309,11 @@ pub async fn update_model_settings(
         .await
         .map_err(|_e| AppError::InternalError("Failed to get models".to_string()))?;
 
-    let effective_backend =
-        effective_embedding_backend(&new_settings, &state.config).as_str().to_string();
-    let show_fastembed_models =
-        is_fastembed_admin_context(&new_settings, &state.config).then(list_fastembed_embedding_models);
+    let effective_backend = effective_embedding_backend(&new_settings, &state.config)
+        .as_str()
+        .to_string();
+    let show_fastembed_models = is_fastembed_admin_context(&new_settings, &state.config)
+        .then(list_fastembed_embedding_models);
 
     Ok(TemplateResponse::new_partial(
         "admin/sections/models.html",
@@ -368,8 +369,8 @@ mod tests {
             embedding_model: Some("Xenova/bge-base-en-v1.5".into()),
             embedding_dimensions: None,
         };
-        let plan = plan_embedding_settings_update(&current, &input, &AppConfig::default())
-            .expect("plan");
+        let plan =
+            plan_embedding_settings_update(&current, &input, &AppConfig::default()).expect("plan");
         assert_eq!(plan.embedding_model, "Xenova/bge-base-en-v1.5");
         assert_eq!(plan.embedding_dimensions, 768);
         assert!(plan.reembedding_needed);
@@ -407,9 +408,7 @@ pub struct SystemPromptEditData {
     default_query_prompt: String,
 }
 
-pub async fn show_edit_system_prompt(
-    State(state): State<HtmlState>,
-) -> TemplateResult {
+pub async fn show_edit_system_prompt(State(state): State<HtmlState>) -> TemplateResult {
     let settings = SystemSettings::get_current(&state.db).await?;
 
     Ok(TemplateResponse::new_template(
@@ -457,9 +456,7 @@ pub struct IngestionPromptEditData {
     default_ingestion_prompt: String,
 }
 
-pub async fn show_edit_ingestion_prompt(
-    State(state): State<HtmlState>,
-) -> TemplateResult {
+pub async fn show_edit_ingestion_prompt(State(state): State<HtmlState>) -> TemplateResult {
     let settings = SystemSettings::get_current(&state.db).await?;
 
     Ok(TemplateResponse::new_template(
@@ -502,9 +499,7 @@ pub struct ImagePromptEditData {
     default_image_prompt: String,
 }
 
-pub async fn show_edit_image_prompt(
-    State(state): State<HtmlState>,
-) -> TemplateResult {
+pub async fn show_edit_image_prompt(State(state): State<HtmlState>) -> TemplateResult {
     let settings = SystemSettings::get_current(&state.db).await?;
 
     Ok(TemplateResponse::new_template(

html-router/src/routes/auth/signup.rs

@@ -2,7 +2,10 @@ use axum::{extract::State, Form};
 use axum_htmx::HxBoosted;
 use serde::{Deserialize, Serialize};
 
-use common::{error::AppError, storage::types::user::{Theme, User}};
+use common::{
+    error::AppError,
+    storage::types::user::{Theme, User},
+};
 
 use crate::{
     html_state::HtmlState,

html-router/src/routes/chat/chat_handlers.rs

@@ -18,8 +18,8 @@ use crate::{
     middlewares::{
         auth_middleware::RequireUser,
         response_middleware::{
-            template_as_response, template_with_headers, TemplateResponse, TemplateResult,
-            ResponseResult,
+            template_as_response, template_with_headers, ResponseResult, TemplateResponse,
+            TemplateResult,
         },
     },
 };

html-router/src/routes/chat/message_response_stream.rs

@@ -22,8 +22,8 @@ use retrieval_pipeline::answer_retrieval::{
 };
 use serde::{Deserialize, Serialize};
 use serde_json::from_str;
-use tokio::sync::Mutex;
 use tokio::sync::mpsc::channel;
+use tokio::sync::Mutex;
 use tracing::{debug, error, info};
 
 use common::storage::{
@@ -36,10 +36,7 @@ use common::storage::{
     },
 };
 
-use crate::{
-    html_state::HtmlState,
-    middlewares::auth_middleware::RequireUser,
-};
+use crate::{html_state::HtmlState, middlewares::auth_middleware::RequireUser};
 
 use super::reference_validation::{collect_reference_ids_from_retrieval, validate_references};
 

html-router/src/routes/chat/mod.rs

@@ -3,15 +3,11 @@ mod message_response_stream;
 mod reference_validation;
 mod references;
 
-use axum::{
-    extract::FromRef,
-    routing::get,
-    Router,
-};
+use axum::{extract::FromRef, routing::get, Router};
 pub use chat_handlers::{
     delete_conversation, new_chat_user_message, new_user_message, patch_conversation_title,
-    reload_sidebar, show_conversation_editing_title,
-    show_chat_base as show_base, show_existing_chat as show_existing,
+    reload_sidebar, show_chat_base as show_base, show_conversation_editing_title,
+    show_existing_chat as show_existing,
 };
 use message_response_stream::get_response_stream;
 use references::show_reference_tooltip;

html-router/src/routes/chat/references.rs

@@ -1,8 +1,6 @@
 #![allow(clippy::missing_docs_in_private_items)]
 
-use axum::{
-    extract::{Path, State},
-};
+use axum::extract::{Path, State};
 use chrono::{DateTime, Utc};
 use chrono_tz::Tz;
 use serde::Serialize;

html-router/src/routes/index/handlers.rs

@@ -13,7 +13,7 @@ use crate::{
     middlewares::{
         auth_middleware::RequireUser,
         response_middleware::{
-            template_as_response, TemplateResponse, TemplateResult, ResponseResult,
+            template_as_response, ResponseResult, TemplateResponse, TemplateResult,
         },
     },
     utils::text_content_preview::truncate_text_contents,
@@ -84,7 +84,8 @@ pub async fn delete_text_content(
     // Delete the text content and any related data
     TextChunk::delete_by_source_id(&text_content.id, &state.db).await?;
     KnowledgeEntity::delete_by_source_id(&text_content.id, &state.db).await?;
-    KnowledgeRelationship::delete_relationships_by_source_id(&text_content.id, &user.id, &state.db).await?;
+    KnowledgeRelationship::delete_relationships_by_source_id(&text_content.id, &user.id, &state.db)
+        .await?;
     state
         .db
         .delete_item::<TextContent>(&text_content.id)

html-router/src/routes/ingestion/handlers.rs

@@ -63,9 +63,7 @@ pub async fn show_ingest_form(
     ))
 }
 
-pub async fn hide_ingest_form(
-    RequireUser(_user): RequireUser,
-) -> TemplateResult {
+pub async fn hide_ingest_form(RequireUser(_user): RequireUser) -> TemplateResult {
     Ok(TemplateResponse::new_template(
         "ingestion/add_content_button.html",
         (),
@@ -148,8 +146,7 @@ pub async fn process_ingest_form(
         user.id.clone(),
     )?;
 
-    let tasks =
-        IngestionTask::create_all_and_add_to_db(payloads, &user.id, &state.db).await?;
+    let tasks = IngestionTask::create_all_and_add_to_db(payloads, &user.id, &state.db).await?;
 
     Ok(TemplateResponse::new_template(
         "dashboard/current_task.html",

html-router/src/routes/knowledge/handlers.rs

@@ -37,7 +37,7 @@ use crate::{
     middlewares::{
         auth_middleware::RequireUser,
         response_middleware::{
-            template_with_headers, TemplateResponse, TemplateResult, ResponseResult,
+            template_with_headers, ResponseResult, TemplateResponse, TemplateResult,
         },
     },
     utils::pagination::{paginate_items, paginate_slice, Pagination},
@@ -185,8 +185,7 @@ pub async fn create_knowledge_entity(
     let description = form.description.trim().to_string();
     let entity_type = KnowledgeEntityType::from(form.entity_type.trim().to_string());
 
-    let embedding_input =
-        format!("name: {name}, description: {description}, type: {entity_type:?}");
+    let embedding_input = KnowledgeEntity::embedding_input_text(&name, &description, entity_type);
     let embedding = state
         .embedding_provider
         .embed(&embedding_input)
@@ -290,10 +289,12 @@ pub async fn suggest_knowledge_relationships(
     if !query_parts.is_empty() {
         let name = form.name.as_deref().unwrap_or("").trim();
         let description = form.description.as_deref().unwrap_or("").trim();
-        let entity_type = form.entity_type.as_deref().map_or(
-            KnowledgeEntityType::Document,
-            |value| KnowledgeEntityType::from(value.to_string()),
-        );
+        let entity_type = form
+            .entity_type
+            .as_deref()
+            .map_or(KnowledgeEntityType::Document, |value| {
+                KnowledgeEntityType::from(value.to_string())
+            });
 
         let suggested = suggest_related_entities(
             &state.db,
@@ -374,10 +375,8 @@ async fn suggest_related_entities(
     draft: DraftEntityQuery<'_>,
     entity_lookup: &HashMap<String, KnowledgeEntity>,
 ) -> Result<HashMap<String, f32>, AppError> {
-    let embedding_input = format!(
-        "name: {}, description: {}, type: {:?}",
-        draft.name, draft.description, draft.entity_type
-    );
+    let embedding_input =
+        KnowledgeEntity::embedding_input_text(draft.name, draft.description, draft.entity_type);
     let embedding = embedding_provider.embed(&embedding_input).await?;
 
     let take = MAX_RELATIONSHIP_SUGGESTIONS * 2;
@@ -484,11 +483,7 @@ fn build_relationship_options(
 
 fn build_relationship_rows(
     relationships: Vec<KnowledgeRelationship>,
-) -> (
-    Vec<RelationshipTableRow>,
-    Vec<String>,
-    String,
-) {
+) -> (Vec<RelationshipTableRow>, Vec<String>, String) {
     let relationship_type_options = collect_relationship_type_options(&relationships);
     let mut frequency: HashMap<String, usize> = HashMap::new();
     let relationships = relationships
@@ -509,10 +504,7 @@ fn build_relationship_rows(
     let default_relationship_type = frequency
         .into_iter()
         .max_by_key(|(_, count)| *count)
-        .map_or_else(
-            || DEFAULT_RELATIONSHIP_TYPE.to_string(),
-            |(label, _)| label,
-        );
+        .map_or_else(|| DEFAULT_RELATIONSHIP_TYPE.to_string(), |(label, _)| label);
 
     (
         relationships,

html-router/src/routes/scratchpad/handlers.rs

@@ -12,7 +12,7 @@ use crate::html_state::HtmlState;
 use crate::middlewares::{
     auth_middleware::RequireUser,
     response_middleware::{
-        template_with_headers, TemplateResponse, TemplateResult, ResponseResult,
+        template_with_headers, ResponseResult, TemplateResponse, TemplateResult,
     },
 };
 use common::storage::types::{

html-router/src/routes/search/handlers.rs

@@ -1,11 +1,11 @@
 use std::collections::HashSet;
 
-use axum::{
-    extract::{Query, State},
-};
+use axum::extract::{Query, State};
 use axum_htmx::{HxBoosted, HxRequest};
 use common::storage::types::{text_content::TextContent, user::User};
-use retrieval_pipeline::{retrieve, RetrievalConfig, RetrievalOutput, RetrievedChunk, RetrievedEntity};
+use retrieval_pipeline::{
+    retrieve, RetrievalConfig, RetrievalOutput, RetrievedChunk, RetrievedEntity,
+};
 use serde::{de, Deserialize, Deserializer, Serialize};
 use std::{fmt, str::FromStr};
 
@@ -48,9 +48,7 @@ impl<'de> Deserialize<'de> for SearchView {
             Some("chunks") => SearchView::Chunks,
             Some("entities") => SearchView::Entities,
             Some(other) => {
-                return Err(de::Error::custom(format!(
-                    "invalid search view: {other}"
-                )));
+                return Err(de::Error::custom(format!("invalid search view: {other}")));
             }
         })
     }
@@ -121,13 +119,12 @@ pub async fn search_result_handler(
     HxBoosted(is_boosted): HxBoosted,
 ) -> TemplateResult {
     let view = params.view;
-    let (search_results_for_template, final_query_param_for_template) = if let Some(actual_query) =
-        params.query
-    {
-        perform_search(&state, &user, actual_query, view).await?
-    } else {
-        (Vec::<SearchResultForTemplate>::new(), String::new())
-    };
+    let (search_results_for_template, final_query_param_for_template) =
+        if let Some(actual_query) = params.query {
+            perform_search(&state, &user, actual_query, view).await?
+        } else {
+            (Vec::<SearchResultForTemplate>::new(), String::new())
+        };
 
     let data = AnswerData {
         search_result: search_results_for_template,

html-router/src/routes/search/mod.rs

@@ -2,9 +2,7 @@ mod handlers;
 
 use axum::{extract::FromRef, routing::get, Router};
 #[allow(clippy::module_name_repetitions)]
-pub use handlers::{
-    search_result_handler as result_handler, SearchParams as SearchQueryParams,
-};
+pub use handlers::{search_result_handler as result_handler, SearchParams as SearchQueryParams};
 
 use crate::html_state::HtmlState;
 

html-router/src/utils/pagination.rs

@@ -31,8 +31,16 @@ impl Pagination {
         } else {
             0
         };
-        let start_index = if page_len == 0 { 0 } else { offset.saturating_add(1) };
-        let end_index = if page_len == 0 { 0 } else { offset.saturating_add(page_len) };
+        let start_index = if page_len == 0 {
+            0
+        } else {
+            offset.saturating_add(1)
+        };
+        let end_index = if page_len == 0 {
+            0
+        } else {
+            offset.saturating_add(page_len)
+        };
 
         Self {
             current_page,
@@ -109,7 +117,11 @@ pub fn paginate_items<T>(
     let total_pages = if total_items == 0 {
         0
     } else {
-        total_items.saturating_sub(1).checked_div(per_page).unwrap_or(0).saturating_add(1)
+        total_items
+            .saturating_sub(1)
+            .checked_div(per_page)
+            .unwrap_or(0)
+            .saturating_add(1)
     };
 
     let mut current_page = requested_page.unwrap_or(1);