fix: arc-share retrieved chunks, centralize entity embeddings, and trim hot-path clones.

ingestion-pipeline/src/pipeline/enrichment_result.rs

@@ -6,11 +6,9 @@ use serde::{Deserialize, Serialize};
 
 use common::{
     error::AppError,
-    storage::{
-        types::{
-            knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
-            knowledge_relationship::KnowledgeRelationship,
-        },
+    storage::types::{
+        knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
+        knowledge_relationship::KnowledgeRelationship,
     },
     utils::embedding::EmbeddingProvider,
 };
@@ -83,25 +81,32 @@ impl LLMEnrichmentResult {
         entity_concurrency: usize,
         embedding_provider: &EmbeddingProvider,
     ) -> Result<Vec<EmbeddedKnowledgeEntity>, AppError> {
-        stream::iter(self.knowledge_entities.clone().into_iter().map(|entity| {
-            let mapper = Arc::clone(&mapper);
-            let source_id = source_id.to_string();
-            let user_id = user_id.to_string();
+        let tasks: Vec<_> = self
+            .knowledge_entities
+            .iter()
+            .map(|entity| {
+                let llm_entity = entity.clone();
+                let mapper = Arc::clone(&mapper);
+                let source_id = source_id.to_string();
+                let user_id = user_id.to_string();
 
-            async move {
-                create_single_entity(
-                    &entity,
-                    &source_id,
-                    &user_id,
-                    mapper,
-                    embedding_provider,
-                )
-                .await
-            }
-        }))
-        .buffer_unordered(entity_concurrency.max(1))
-        .try_collect()
-        .await
+                async move {
+                    create_single_entity(
+                        llm_entity,
+                        &source_id,
+                        &user_id,
+                        mapper,
+                        embedding_provider,
+                    )
+                    .await
+                }
+            })
+            .collect();
+
+        stream::iter(tasks)
+            .buffer_unordered(entity_concurrency.max(1))
+            .try_collect()
+            .await
     }
 
     fn process_relationships(
@@ -129,7 +134,7 @@ impl LLMEnrichmentResult {
 }
 
 async fn create_single_entity(
-    llm_entity: &LLMKnowledgeEntity,
+    llm_entity: LLMKnowledgeEntity,
     source_id: &str,
     user_id: &str,
     mapper: Arc<GraphMapper>,
@@ -137,9 +142,11 @@ async fn create_single_entity(
 ) -> Result<EmbeddedKnowledgeEntity, AppError> {
     let assigned_id = mapper.get_id(&llm_entity.key)?.to_string();
 
-    let embedding_input = format!(
-        "name: {}, description: {}, type: {}",
-        llm_entity.name, llm_entity.description, llm_entity.entity_type
+    let entity_type = KnowledgeEntityType::from(llm_entity.entity_type);
+    let embedding_input = KnowledgeEntity::embedding_input_text(
+        &llm_entity.name,
+        &llm_entity.description,
+        entity_type,
     );
 
     let embedding = embedding_provider.embed(&embedding_input).await?;
@@ -149,9 +156,9 @@ async fn create_single_entity(
         id: assigned_id,
         created_at: now,
         updated_at: now,
-        name: llm_entity.name.clone(),
-        description: llm_entity.description.clone(),
-        entity_type: KnowledgeEntityType::from(llm_entity.entity_type.clone()),
+        name: llm_entity.name,
+        description: llm_entity.description,
+        entity_type,
         source_id: source_id.to_string(),
         metadata: None,
         user_id: user_id.into(),

ingestion-pipeline/src/pipeline/persistence.rs

@@ -48,20 +48,20 @@ const STORE_RELATIONSHIPS: &str = r"
 pub(super) async fn store_vector_chunks(
     db: &SurrealDbClient,
     task_id: &str,
-    chunks: &[EmbeddedTextChunk],
+    chunks: Vec<EmbeddedTextChunk>,
 ) -> Result<usize, AppError> {
+    let chunk_count = chunks.len();
     for embedded in chunks {
-        TextChunk::store_with_embedding(embedded.chunk.clone(), embedded.embedding.clone(), db)
-            .await?;
         debug!(
             task_id = %task_id,
             chunk_id = %embedded.chunk.id,
             chunk_len = embedded.chunk.chunk.chars().count(),
             "chunk persisted"
         );
+        TextChunk::store_with_embedding(embedded.chunk, embedded.embedding, db).await?;
     }
 
-    Ok(chunks.len())
+    Ok(chunk_count)
 }
 
 /// Persists knowledge entities and their relationships.

ingestion-pipeline/src/pipeline/stages.rs

@@ -155,7 +155,7 @@ pub async fn persist(
     let entity_count = entities.len();
     let relationship_count = relationships.len();
 
-    let chunk_count = store_vector_chunks(ctx.db, ctx.task_id.as_str(), &chunks).await?;
+    let chunk_count = store_vector_chunks(ctx.db, ctx.task_id.as_str(), chunks).await?;
     store_graph_entities(ctx.db, &ctx.pipeline_config.tuning, entities, relationships).await?;
     ctx.db.store_item(text_content).await?;
     rebuild(ctx.db).await?;

ingestion-pipeline/src/pipeline/tests.rs

@@ -92,7 +92,7 @@ impl MockServices {
                 entity: retrieved_entity,
                 score: 0.8,
                 chunks: std::sync::Arc::new(vec![RetrievedChunk {
-                    chunk: retrieved_chunk,
+                    chunk: std::sync::Arc::new(retrieved_chunk),
                     score: 0.7,
                 }]),
             }],

ingestion-pipeline/src/utils/file_text_extraction.rs

@@ -74,10 +74,7 @@ pub async fn extract_text_from_file(
     config: &AppConfig,
     storage: &StorageManager,
 ) -> Result<String, AppError> {
-    let file_bytes = storage
-        .get(&file_info.path)
-        .await
-        .map_err(AppError::from)?;
+    let file_bytes = storage.get(&file_info.path).await.map_err(AppError::from)?;
     let local_path = resolve_existing_local_path(storage, &file_info.path).await;
 
     match file_info.mime_type.as_str() {