starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-07-02 11:01:38 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

chore: refactor retrieval pipeline to chunk-first RRF with derived entities and slimmer eval surface.

Browse Source 
Collapse the multi-strategy entity engine into one benchmarked chunk retrieval path, derive entities from retrieved chunks, and update consumers, docs, and clippy fixes across the workspace.
This commit is contained in:
Per Stark
2026-05-30 22:19:08 +02:00
parent c70141de35
commit 5c2d2e24d3
38 changed files with 1049 additions and 2614 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evaluations/src/pipeline/stages/run_queries.rs
+7 -23
View File
@@ -6,7 +6,7 @@ use futures::stream::{self, StreamExt};
use tracing::{debug, info};
use crate::eval::{
adapt_strategy_output, build_case_diagnostics, text_contains_answer, CaseDiagnostics,
adapt_retrieval_output, build_case_diagnostics, text_contains_answer, CaseDiagnostics,
CaseSummary, RetrievedSummary,
};
use retrieval_pipeline::{
@@ -51,14 +51,8 @@ pub(crate) async fn run_queries(
None
};
let mut retrieval_config = RetrievalConfig {
strategy: config.retrieval.strategy,
..Default::default()
};
let mut retrieval_config = RetrievalConfig::default();
retrieval_config.tuning.rerank_keep_top = config.retrieval.rerank_keep_top;
if retrieval_config.tuning.fallback_min_results < config.retrieval.rerank_keep_top {
retrieval_config.tuning.fallback_min_results = config.retrieval.rerank_keep_top;
}
retrieval_config.tuning.chunk_result_cap = config.retrieval.chunk_result_cap.max(1);
if let Some(value) = config.retrieval.chunk_vector_take {
retrieval_config.tuning.chunk_vector_take = value;
@@ -81,9 +75,6 @@ pub(crate) async fn run_queries(
if let Some(value) = config.retrieval.chunk_rrf_use_fts {
retrieval_config.tuning.flags.chunk_rrf_use_fts = value.into();
}
if let Some(value) = config.retrieval.chunk_avg_chars_per_token {
retrieval_config.tuning.avg_chars_per_token = value;
}
if let Some(value) = config.retrieval.max_chunks_per_entity {
retrieval_config.tuning.max_chunks_per_entity = value;
}
@@ -187,7 +178,7 @@ pub(crate) async fn run_queries(
None => None,
};
let params = pipeline::StrategyParams {
let params = pipeline::RetrievalParams {
db_client: &db,
openai_client: &openai_client,
embedding_provider: Some(&embedding_provider),
@@ -196,26 +187,19 @@ pub(crate) async fn run_queries(
config: (*retrieval_config).clone(),
reranker,
};
let (result_output, pipeline_diagnostics, stage_timings) = if diagnostics_enabled {
let outcome = pipeline::run_pipeline_with_embedding_with_diagnostics(
let (result_output, pipeline_diagnostics, stage_timings) = {
let outcome = pipeline::run_with_embedding_instrumented(
params,
query_embedding,
diagnostics_enabled,
)
.await
.with_context(|| format!("running pipeline for question {question_id}"))?;
(outcome.results, outcome.diagnostics, outcome.stage_timings)
} else {
let outcome = pipeline::run_pipeline_with_embedding_with_metrics(
params,
query_embedding,
)
.await
.with_context(|| format!("running pipeline for question {question_id}"))?;
(outcome.results, None, outcome.stage_timings)
};
let query_latency = query_start.elapsed().as_millis();
let candidates = adapt_strategy_output(result_output);
let candidates = adapt_retrieval_output(result_output);
let mut retrieved = Vec::new();
let mut match_rank = None;
let answers_lower: Vec<String> =
evaluations/src/pipeline/stages/summarize.rs
+4 -2
View File
@@ -201,7 +201,10 @@ pub(crate) async fn summarize(
rerank_keep_top: config.retrieval.rerank_keep_top,
concurrency: config.concurrency.max(1),
detailed_report: config.detailed_report,
retrieval_strategy: config.retrieval.strategy.to_string(),
resolve_entities: ctx
.retrieval_config
.as_ref()
.is_some_and(|config| config.resolve_entities),
chunk_result_cap: config.retrieval.chunk_result_cap,
chunk_rrf_k: active_tuning.chunk_rrf_k,
chunk_rrf_vector_weight: active_tuning.chunk_rrf_vector_weight,
@@ -214,7 +217,6 @@ pub(crate) async fn summarize(
ingest_chunk_overlap_tokens: config.ingest.ingest_chunk_overlap_tokens,
chunk_vector_take: active_tuning.chunk_vector_take,
chunk_fts_take: active_tuning.chunk_fts_take,
chunk_avg_chars_per_token: active_tuning.avg_chars_per_token,
max_chunks_per_entity: active_tuning.max_chunks_per_entity,
cases: summaries,
});