starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-07-15 01:02:47 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

fix: improved storage manager, prep for s3

Browse Source
This commit is contained in:
Per Stark
2025-11-03 12:39:15 +01:00
parent f13791cfcf
commit 3196e65172
20 changed files with 1438 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ingestion-pipeline/src/pipeline/mod.rs
+3
View File
@@ -19,6 +19,7 @@ use common::{
error::AppError,
storage::{
db::SurrealDbClient,
store::StorageManager,
types::{
ingestion_payload::IngestionPayload,
ingestion_task::{IngestionTask, TaskErrorInfo},
@@ -47,12 +48,14 @@ impl IngestionPipeline {
openai_client: Arc<Client<async_openai::config::OpenAIConfig>>,
config: AppConfig,
reranker_pool: Option<Arc<RerankerPool>>,
storage: StorageManager,
) -> Result<Self, AppError> {
let services = DefaultPipelineServices::new(
db.clone(),
openai_client.clone(),
config.clone(),
reranker_pool,
storage,
);
Self::with_services(db, IngestionConfig::default(), Arc::new(services))
ingestion-pipeline/src/pipeline/preparation.rs
+5 -2
View File
@@ -2,6 +2,7 @@ use common::{
error::AppError,
storage::{
db::SurrealDbClient,
store::StorageManager,
types::{
ingestion_payload::IngestionPayload,
text_content::{TextContent, UrlInfo},
@@ -19,6 +20,7 @@ pub(crate) async fn to_text_content(
db: &SurrealDbClient,
config: &AppConfig,
openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
storage: &StorageManager,
) -> Result<TextContent, AppError> {
match ingestion_payload {
IngestionPayload::Url {
@@ -27,7 +29,7 @@ pub(crate) async fn to_text_content(
category,
user_id,
} => {
let (article, file_info) = extract_text_from_url(&url, db, &user_id, config).await?;
let (article, file_info) = extract_text_from_url(&url, db, &user_id, storage).await?;
Ok(TextContent::new(
article.text_content.into(),
Some(context),
@@ -60,7 +62,8 @@ pub(crate) async fn to_text_content(
category,
user_id,
} => {
let text = extract_text_from_file(&file_info, db, openai_client, config).await?;
let text =
extract_text_from_file(&file_info, db, openai_client, config, storage).await?;
Ok(TextContent::new(
text,
Some(context),
ingestion-pipeline/src/pipeline/services.rs
+12 -1
View File
@@ -10,6 +10,7 @@ use common::{
error::AppError,
storage::{
db::SurrealDbClient,
store::StorageManager,
types::{
ingestion_payload::IngestionPayload, knowledge_entity::KnowledgeEntity,
knowledge_relationship::KnowledgeRelationship, system_settings::SystemSettings,
@@ -65,6 +66,7 @@ pub struct DefaultPipelineServices {
openai_client: Arc<async_openai::Client<async_openai::config::OpenAIConfig>>,
config: AppConfig,
reranker_pool: Option<Arc<RerankerPool>>,
storage: StorageManager,
}
impl DefaultPipelineServices {
@@ -73,12 +75,14 @@ impl DefaultPipelineServices {
openai_client: Arc<async_openai::Client<async_openai::config::OpenAIConfig>>,
config: AppConfig,
reranker_pool: Option<Arc<RerankerPool>>,
storage: StorageManager,
) -> Self {
Self {
db,
openai_client,
config,
reranker_pool,
storage,
}
}
@@ -144,7 +148,14 @@ impl PipelineServices for DefaultPipelineServices {
&self,
payload: IngestionPayload,
) -> Result<TextContent, AppError> {
to_text_content(payload, &self.db, &self.config, &self.openai_client).await
to_text_content(
payload,
&self.db,
&self.config,
&self.openai_client,
&self.storage,
)
.await
}
async fn retrieve_similar_entities(