retrieval simplfied

evaluations/src/corpus/mod.rs

@@ -14,13 +14,13 @@ pub use store::{
 };
 
 pub fn make_ingestion_config(config: &crate::args::Config) -> ingestion_pipeline::IngestionConfig {
-    let mut tuning = ingestion_pipeline::IngestionTuning::default();
-    tuning.chunk_min_tokens = config.ingest.ingest_chunk_min_tokens;
-    tuning.chunk_max_tokens = config.ingest.ingest_chunk_max_tokens;
-    tuning.chunk_overlap_tokens = config.ingest.ingest_chunk_overlap_tokens;
-
     ingestion_pipeline::IngestionConfig {
-        tuning,
+        tuning: ingestion_pipeline::IngestionTuning {
+            chunk_min_tokens: config.ingest.ingest_chunk_min_tokens,
+            chunk_max_tokens: config.ingest.ingest_chunk_max_tokens,
+            chunk_overlap_tokens: config.ingest.ingest_chunk_overlap_tokens,
+            ..Default::default()
+        },
         chunk_only: config.ingest.ingest_chunks_only,
     }
 }

evaluations/src/corpus/orchestrator.rs

@@ -106,6 +106,7 @@ struct IngestionStats {
     negative_ingested: usize,
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn ensure_corpus(
     dataset: &ConvertedDataset,
     slice: &ResolvedSlice<'_>,
@@ -337,11 +338,9 @@ pub async fn ensure_corpus(
         });
     }
 
-    for record in &mut records {
-        if let Some(ref mut entry) = record {
-            if entry.dirty {
-                store.persist(&entry.shard)?;
-            }
+    for entry in records.iter_mut().flatten() {
+        if entry.dirty {
+            store.persist(&entry.shard)?;
         }
     }
 
@@ -403,6 +402,7 @@ pub async fn ensure_corpus(
     Ok(handle)
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn ingest_paragraph_batch(
     dataset: &ConvertedDataset,
     targets: &[IngestRequest<'_>],
@@ -430,8 +430,10 @@ async fn ingest_paragraph_batch(
         .await
         .context("applying migrations for ingestion")?;
 
-    let mut app_config = AppConfig::default();
-    app_config.storage = StorageKind::Memory;
+    let app_config = AppConfig {
+        storage: StorageKind::Memory,
+        ..Default::default()
+    };
     let backend: DynStore = Arc::new(InMemory::new());
     let storage = StorageManager::with_backend(backend, StorageKind::Memory);
 
@@ -444,8 +446,7 @@ async fn ingest_paragraph_batch(
         storage,
         embedding.clone(),
         pipeline_config,
-    )
-    .await?;
+    )?;
     let pipeline = Arc::new(pipeline);
 
     let mut shards = Vec::with_capacity(targets.len());
@@ -454,7 +455,7 @@ async fn ingest_paragraph_batch(
         info!(
             batch = batch_index,
             batch_size = batch.len(),
-            total_batches = (targets.len() + batch_size - 1) / batch_size,
+            total_batches = targets.len().div_ceil(batch_size),
             "Ingesting paragraph batch"
         );
         let model_clone = embedding_model.clone();
@@ -486,6 +487,7 @@ async fn ingest_paragraph_batch(
     Ok(shards)
 }
 
+#[allow(clippy::too_many_arguments)]
 async fn ingest_single_paragraph(
     pipeline: Arc<IngestionPipeline>,
     request: IngestRequest<'_>,

evaluations/src/corpus/store.rs

@@ -481,6 +481,7 @@ impl ParagraphShardStore {
 }
 
 impl ParagraphShard {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         paragraph: &ConvertedParagraph,
         shard_path: String,
@@ -674,10 +675,8 @@ async fn execute_batched_inserts<T: Clone + Serialize + 'static>(
 
         let slice = &batches[start..group_end];
         let mut query = db.client.query("BEGIN TRANSACTION;");
-        let mut bind_index = 0usize;
-        for batch in slice {
+        for (bind_index, batch) in slice.iter().enumerate() {
             let name = format!("{prefix}{bind_index}");
-            bind_index += 1;
             query = query
                 .query(format!("{} ${};", statement.as_ref(), name))
                 .bind((name, batch.items.clone()));
@@ -702,7 +701,7 @@ async fn execute_batched_inserts<T: Clone + Serialize + 'static>(
 pub async fn seed_manifest_into_db(db: &SurrealDbClient, manifest: &CorpusManifest) -> Result<()> {
     let batches = build_manifest_batches(manifest).context("preparing manifest batches")?;
 
-    let result = (|| async {
+    let result = async {
         execute_batched_inserts(
             db,
             format!("INSERT INTO {}", TextContent::table_name()),
@@ -752,7 +751,7 @@ pub async fn seed_manifest_into_db(db: &SurrealDbClient, manifest: &CorpusManife
         .await?;
 
         Ok(())
-    })()
+    }
     .await;
 
     if result.is_err() {
@@ -778,7 +777,6 @@ pub async fn seed_manifest_into_db(db: &SurrealDbClient, manifest: &CorpusManife
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::db_helpers::change_embedding_length_in_hnsw_indexes;
     use chrono::Utc;
     use common::storage::types::knowledge_entity::KnowledgeEntityType;
     use uuid::Uuid;
@@ -905,9 +903,6 @@ mod tests {
         db.apply_migrations()
             .await
             .expect("apply migrations for memory db");
-        change_embedding_length_in_hnsw_indexes(&db, 3)
-            .await
-            .expect("set embedding index dimension for test");
 
         let manifest = build_manifest();
         seed_manifest_into_db(&db, &manifest)