fix: schedule nightly index rebuild on worker and skip per-ingest rebuild.

Ingest relies on SurrealDB incremental index maintenance; the worker runs native REBUILD INDEX on a configurable interval with lease state on system_settings.
This commit is contained in:
Per Stark
2026-06-12 15:01:53 +02:00
parent 4e8a58fff1
commit ead17530bd
12 changed files with 370 additions and 11 deletions
+7 -1
View File
@@ -43,6 +43,7 @@ async fn main() -> anyhow::Result<()> {
let worker_openai = Arc::clone(&services.openai_client);
let worker_embedding = Arc::clone(&services.embedding_provider);
let worker_config = services.config.clone();
let index_rebuild_interval_secs = worker_config.index_rebuild_interval_secs;
let worker_reranker = services.reranker_pool.clone();
let worker_storage = services.storage.clone();
@@ -59,7 +60,12 @@ async fn main() -> anyhow::Result<()> {
worker_embedding,
)?);
run_worker_loop(worker_db, ingestion_pipeline).await
run_worker_loop(
worker_db,
ingestion_pipeline,
index_rebuild_interval_secs,
)
.await
});
tokio::select! {
+9 -2
View File
@@ -26,7 +26,12 @@ async fn main() -> anyhow::Result<()> {
Arc::clone(&services.embedding_provider),
)?);
run_worker_loop(services.db, ingestion_pipeline).await
run_worker_loop(
services.db,
ingestion_pipeline,
services.config.index_rebuild_interval_secs,
)
.await
}
#[cfg(test)]
@@ -69,7 +74,9 @@ mod tests {
let db = Arc::clone(&services.db);
let pipeline = Arc::new(pipeline);
let worker =
tokio::spawn(async move { ingestion_pipeline::run_worker_loop(db, pipeline).await });
tokio::spawn(async move {
ingestion_pipeline::run_worker_loop(db, pipeline, 0).await
});
tokio::time::sleep(Duration::from_millis(250)).await;
assert!(