chore: refactor retrieval pipeline to chunk-first RRF with derived entities and slimmer eval surface.

Collapse the multi-strategy entity engine into one benchmarked chunk retrieval path, derive entities from retrieved chunks, and update consumers, docs, and clippy fixes across the workspace.

html-router/src/html_state.rs

@@ -2,10 +2,7 @@ use common::storage::types::conversation::SidebarConversation;
 use common::storage::{db::SurrealDbClient, store::StorageManager};
 use common::utils::embedding::EmbeddingProvider;
 use common::utils::template_engine::{ProvidesTemplateEngine, TemplateEngine};
-use common::{
-    create_template_engine, storage::db::ProvidesDb,
-    utils::config::{AppConfig, RetrievalStrategy},
-};
+use common::{create_template_engine, storage::db::ProvidesDb, utils::config::AppConfig};
 use retrieval_pipeline::reranking::RerankerPool;
 use std::collections::HashMap;
 use std::sync::{
@@ -75,10 +72,6 @@ impl HtmlState {
         }
     }
 
-    pub fn retrieval_strategy(&self) -> RetrievalStrategy {
-        self.config.resolved_retrieval_strategy()
-    }
-
     pub async fn get_cached_conversation_archive(
         &self,
         user_id: &str,

html-router/src/routes/chat/message_response_stream.rs

@@ -16,12 +16,9 @@ use futures::{
 };
 use json_stream_parser::JsonStreamParser;
 use minijinja::Value;
-use retrieval_pipeline::{
-    answer_retrieval::{
-        chunks_to_chat_context, create_chat_request, create_user_message_with_history,
-        LLMResponseFormat,
-    },
-    retrieved_entities_to_json,
+use retrieval_pipeline::answer_retrieval::{
+    chunks_to_chat_context, create_chat_request, create_user_message_with_history,
+    LLMResponseFormat,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::from_str;
@@ -189,11 +186,7 @@ struct ReferenceData {
 }
 
 fn extract_reference_strings(response: &LLMResponseFormat) -> Vec<String> {
-    response
-        .references
-        .iter()
-        .map(|reference| reference.reference.clone())
-        .collect()
+    response.reference_ids()
 }
 
 #[allow(clippy::too_many_lines)]
@@ -362,10 +355,9 @@ async fn prepare_chat_request(
         None => None,
     };
 
-    let strategy = state.retrieval_strategy();
-    let config = retrieval_pipeline::RetrievalConfig::for_chat(strategy);
+    let config = retrieval_pipeline::RetrievalConfig::default();
 
-    let retrieval_result = match retrieval_pipeline::retrieve_entities(
+    let retrieval_result = match retrieval_pipeline::retrieve(
         &state.db,
         &state.openai_client,
         Some(&*state.embedding_provider),
@@ -387,12 +379,9 @@ async fn prepare_chat_request(
     let allowed_reference_ids = collect_reference_ids_from_retrieval(&retrieval_result);
 
     let context_json = match retrieval_result {
-        retrieval_pipeline::StrategyOutput::Chunks(chunks) => chunks_to_chat_context(&chunks),
-        retrieval_pipeline::StrategyOutput::Entities(entities) => {
-            retrieved_entities_to_json(&entities)
-        }
-        retrieval_pipeline::StrategyOutput::Search(search_result) => {
-            chunks_to_chat_context(&search_result.chunks)
+        retrieval_pipeline::RetrievalOutput::Chunks(chunks) => chunks_to_chat_context(&chunks),
+        retrieval_pipeline::RetrievalOutput::WithEntities { chunks, .. } => {
+            chunks_to_chat_context(&chunks)
         }
     };
     let formatted_user_message =

html-router/src/routes/chat/reference_validation.rs

@@ -9,7 +9,7 @@ use common::{
         types::{knowledge_entity::KnowledgeEntity, text_chunk::TextChunk, StoredObject},
     },
 };
-use retrieval_pipeline::StrategyOutput;
+use retrieval_pipeline::RetrievalOutput;
 use uuid::Uuid;
 
 pub(crate) const MAX_REFERENCE_COUNT: usize = 10;
@@ -86,40 +86,29 @@ pub(crate) enum ReferenceLookupTarget {
 }
 
 pub(crate) fn collect_reference_ids_from_retrieval(
-    retrieval_result: &StrategyOutput,
+    retrieval_result: &RetrievalOutput,
 ) -> Vec<String> {
     let mut ids = Vec::new();
     let mut seen = HashSet::new();
 
+    let mut push_id = |id: String| {
+        if seen.insert(id.clone()) {
+            ids.push(id);
+        }
+    };
+
     match retrieval_result {
-        StrategyOutput::Chunks(chunks) => {
+        RetrievalOutput::Chunks(chunks) => {
             for chunk in chunks {
-                let id = chunk.chunk.id.clone();
-                if seen.insert(id.clone()) {
-                    ids.push(id);
-                }
+                push_id(chunk.chunk.id.clone());
             }
         }
-        StrategyOutput::Entities(entities) => {
+        RetrievalOutput::WithEntities { chunks, entities } => {
+            for chunk in chunks {
+                push_id(chunk.chunk.id.clone());
+            }
             for entity in entities {
-                let id = entity.entity.id.clone();
-                if seen.insert(id.clone()) {
-                    ids.push(id);
-                }
-            }
-        }
-        StrategyOutput::Search(search) => {
-            for chunk in &search.chunks {
-                let id = chunk.chunk.id.clone();
-                if seen.insert(id.clone()) {
-                    ids.push(id);
-                }
-            }
-            for entity in &search.entities {
-                let id = entity.entity.id.clone();
-                if seen.insert(id.clone()) {
-                    ids.push(id);
-                }
+                push_id(entity.entity.id.clone());
             }
         }
     }

html-router/src/routes/index/handlers.rs

@@ -13,7 +13,7 @@ use crate::{
     middlewares::{
         auth_middleware::RequireUser,
         response_middleware::{
-            template_as_response, HtmlError, TemplateResponse, TemplateResult, ResponseResult,
+            template_as_response, TemplateResponse, TemplateResult, ResponseResult,
         },
     },
     utils::text_content_preview::truncate_text_contents,

html-router/src/routes/knowledge/handlers.rs

@@ -32,7 +32,7 @@ use crate::{
     middlewares::{
         auth_middleware::RequireUser,
         response_middleware::{
-            template_with_headers, HtmlError, TemplateResponse, TemplateResult, ResponseResult,
+            template_with_headers, TemplateResponse, TemplateResult, ResponseResult,
         },
     },
     utils::pagination::{paginate_items, Pagination},
@@ -284,9 +284,9 @@ pub async fn suggest_knowledge_relationships(
             None => None,
         };
 
-        let config = retrieval_pipeline::RetrievalConfig::for_relationship_suggestion();
-        if let Ok(retrieval_pipeline::StrategyOutput::Entities(results)) =
-            retrieval_pipeline::retrieve_entities(
+        let config = retrieval_pipeline::RetrievalConfig::with_entities();
+        if let Ok(retrieval_pipeline::RetrievalOutput::WithEntities { entities, .. }) =
+            retrieval_pipeline::retrieve(
                 &state.db,
                 &state.openai_client,
                 Some(&*state.embedding_provider),
@@ -297,7 +297,7 @@ pub async fn suggest_knowledge_relationships(
             )
             .await
         {
-            for retrieval_pipeline::RetrievedEntity { entity, score, .. } in results {
+            for retrieval_pipeline::RetrievedEntity { entity, score, .. } in entities {
                 if suggestion_scores.len() >= MAX_RELATIONSHIP_SUGGESTIONS {
                     break;
                 }

html-router/src/routes/scratchpad/handlers.rs

@@ -12,7 +12,7 @@ use crate::html_state::HtmlState;
 use crate::middlewares::{
     auth_middleware::RequireUser,
     response_middleware::{
-        template_with_headers, HtmlError, TemplateResponse, TemplateResult, ResponseResult,
+        template_with_headers, TemplateResponse, TemplateResult, ResponseResult,
     },
 };
 use common::storage::types::{

html-router/src/routes/search/handlers.rs

@@ -4,7 +4,7 @@ use axum::{
     extract::{Query, State},
 };
 use common::storage::types::{text_content::TextContent, user::User};
-use retrieval_pipeline::{RetrievalConfig, SearchResult, SearchTarget, StrategyOutput};
+use retrieval_pipeline::{retrieve, RetrievalConfig, RetrievalOutput, RetrievedChunk, RetrievedEntity};
 use serde::{de, Deserialize, Deserializer, Serialize};
 use std::{fmt, str::FromStr};
 
@@ -108,35 +108,35 @@ async fn perform_search(
         return Ok((Vec::new(), String::new()));
     }
 
-    let config = RetrievalConfig::for_search(SearchTarget::Both);
+    let config = RetrievalConfig::with_entities();
 
     let reranker_lease = match &state.reranker_pool {
         Some(pool) => pool.checkout().await,
         None => None,
     };
 
-    let params = retrieval_pipeline::pipeline::StrategyParams {
-        db_client: &state.db,
-        openai_client: &state.openai_client,
-        embedding_provider: Some(&state.embedding_provider),
-        input_text: trimmed_query,
-        user_id: &user.id,
+    let result = retrieve(
+        &state.db,
+        &state.openai_client,
+        Some(&state.embedding_provider),
+        trimmed_query,
+        &user.id,
         config,
-        reranker: reranker_lease,
-    };
-    let result = retrieval_pipeline::pipeline::execute(params).await?;
+        reranker_lease,
+    )
+    .await?;
 
-    let search_result = match result {
-        StrategyOutput::Search(sr) => sr,
-        _ => SearchResult::new(vec![], vec![]),
+    let (chunks, entities) = match result {
+        RetrievalOutput::WithEntities { chunks, entities } => (chunks, entities),
+        RetrievalOutput::Chunks(chunks) => (chunks, Vec::new()),
     };
 
-    let source_label_map = collect_source_label_map(state, user, &search_result).await?;
+    let source_label_map = collect_source_label_map(state, user, &chunks, &entities).await?;
 
     let mut combined_results: Vec<SearchResultForTemplate> =
-        Vec::with_capacity(search_result.chunks.len().saturating_add(search_result.entities.len()));
+        Vec::with_capacity(chunks.len().saturating_add(entities.len()));
 
-    for chunk_result in search_result.chunks {
+    for chunk_result in chunks {
         let source_label = source_label_map
             .get(&chunk_result.chunk.source_id)
             .cloned()
@@ -155,7 +155,7 @@ async fn perform_search(
         });
     }
 
-    for entity_result in search_result.entities {
+    for entity_result in entities {
         let source_label = source_label_map
             .get(&entity_result.entity.source_id)
             .cloned()
@@ -187,13 +187,14 @@ async fn perform_search(
 async fn collect_source_label_map(
     state: &HtmlState,
     user: &User,
-    search_result: &SearchResult,
+    chunks: &[RetrievedChunk],
+    entities: &[RetrievedEntity],
 ) -> Result<std::collections::HashMap<String, String>, HtmlError> {
     let mut source_ids = HashSet::new();
-    for chunk_result in &search_result.chunks {
+    for chunk_result in chunks {
         source_ids.insert(chunk_result.chunk.source_id.clone());
     }
-    for entity_result in &search_result.entities {
+    for entity_result in entities {
         source_ids.insert(entity_result.entity.source_id.clone());
     }