fix: schedule nightly index rebuild on worker and skip per-ingest rebuild.

Ingest relies on SurrealDB incremental index maintenance; the worker runs native REBUILD INDEX on a configurable interval with lease state on system_settings.

ingestion-pipeline/src/lib.rs

@@ -6,6 +6,7 @@ pub mod utils;
 use chrono::Utc;
 use common::storage::{
     db::SurrealDbClient,
+    indexes::maybe_run_scheduled_index_rebuild,
     types::ingestion_task::{IngestionTask, DEFAULT_LEASE_SECS},
 };
 pub use pipeline::{
@@ -25,6 +26,7 @@ const WORKER_CLAIM_ERROR_BACKOFF_MS: u64 = 1_000;
 pub async fn run_worker_loop(
     db: Arc<SurrealDbClient>,
     ingestion_pipeline: Arc<IngestionPipeline>,
+    index_rebuild_interval_secs: u64,
 ) -> anyhow::Result<()> {
     let worker_id = format!("ingestion-worker-{}", Uuid::new_v4());
     let lease_duration = Duration::from_secs(DEFAULT_LEASE_SECS as u64);
@@ -46,6 +48,12 @@ pub async fn run_worker_loop(
                 }
             }
             Ok(None) => {
+                maybe_run_scheduled_index_rebuild(
+                    db.as_ref(),
+                    &worker_id,
+                    index_rebuild_interval_secs,
+                )
+                .await;
                 sleep(idle_backoff).await;
             }
             Err(err) => {

ingestion-pipeline/src/pipeline/stages.rs

@@ -6,10 +6,7 @@
 
 use common::{
     error::AppError,
-    storage::{
-        indexes::rebuild,
-        types::{ingestion_payload::IngestionPayload, system_settings::SystemSettings},
-    },
+    storage::types::{ingestion_payload::IngestionPayload, system_settings::SystemSettings},
 };
 use state_machines::core::GuardError;
 use tracing::{debug, instrument};
@@ -179,7 +176,6 @@ pub async fn persist(
     )
     .await?;
     ctx.db.store_item(text_content).await?;
-    rebuild(ctx.db).await?;
 
     debug!(
         task_id = %ctx.task_id,