starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-04-18 23:19:47 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

fix: improved storage manager, prep for s3

Browse Source
This commit is contained in:
Per Stark
2025-11-03 12:39:15 +01:00
parent f13791cfcf
commit 3196e65172
20 changed files with 1438 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
main/src/worker.rs
View File

@@ -1,6 +1,8 @@
use std::sync::Arc;
use common::{storage::db::SurrealDbClient, utils::config::get_config};
use common::{
storage::db::SurrealDbClient, storage::store::StorageManager, utils::config::get_config,
};
use composite_retrieval::reranking::RerankerPool;
use ingestion_pipeline::{pipeline::IngestionPipeline, run_worker_loop};
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
@@ -35,8 +37,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let reranker_pool = RerankerPool::maybe_from_config(&config)?;
// Create global storage manager
let storage = StorageManager::new(&config).await?;
let ingestion_pipeline = Arc::new(
IngestionPipeline::new(db.clone(), openai_client.clone(), config, reranker_pool).await?,
IngestionPipeline::new(
db.clone(),
openai_client.clone(),
config,
reranker_pool,
storage,
)
.await?,
);
run_worker_loop(db, ingestion_pipeline).await