starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-07-10 23:02:53 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

feat: refactored error handling

Browse Source
This commit is contained in:
Per Stark
2025-01-01 23:26:41 +01:00
parent 796bbc0225
commit 519f6c6eb1
25 changed files with 439 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ingress/analysis/ingress_analyser.rs
+15 -13
View File
@@ -1,13 +1,16 @@
use crate::{
error::ProcessingError,
error::AppError,
ingress::analysis::prompt::{get_ingress_analysis_schema, INGRESS_ANALYSIS_SYSTEM_MESSAGE},
retrieval::combined_knowledge_entity_retrieval,
storage::types::knowledge_entity::KnowledgeEntity,
};
use async_openai::types::{
ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage,
CreateChatCompletionRequest, CreateChatCompletionRequestArgs, ResponseFormat,
ResponseFormatJsonSchema,
use async_openai::{
error::OpenAIError,
types::{
ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage,
CreateChatCompletionRequest, CreateChatCompletionRequestArgs, ResponseFormat,
ResponseFormatJsonSchema,
},
};
use serde_json::json;
use surrealdb::engine::any::Any;
@@ -38,7 +41,7 @@ impl<'a> IngressAnalyzer<'a> {
instructions: &str,
text: &str,
user_id: &str,
) -> Result<LLMGraphAnalysisResult, ProcessingError> {
) -> Result<LLMGraphAnalysisResult, AppError> {
let similar_entities = self
.find_similar_entities(category, instructions, text, user_id)
.await?;
@@ -53,7 +56,7 @@ impl<'a> IngressAnalyzer<'a> {
instructions: &str,
text: &str,
user_id: &str,
) -> Result<Vec<KnowledgeEntity>, ProcessingError> {
) -> Result<Vec<KnowledgeEntity>, AppError> {
let input_text = format!(
"content: {}, category: {}, user_instructions: {}",
text, category, instructions
@@ -74,7 +77,7 @@ impl<'a> IngressAnalyzer<'a> {
instructions: &str,
text: &str,
similar_entities: &[KnowledgeEntity],
) -> Result<CreateChatCompletionRequest, ProcessingError> {
) -> Result<CreateChatCompletionRequest, OpenAIError> {
let entities_json = json!(similar_entities
.iter()
.map(|entity| {
@@ -114,13 +117,12 @@ impl<'a> IngressAnalyzer<'a> {
])
.response_format(response_format)
.build()
.map_err(|e| ProcessingError::LLMParsingError(e.to_string()))
}
async fn perform_analysis(
&self,
request: CreateChatCompletionRequest,
) -> Result<LLMGraphAnalysisResult, ProcessingError> {
) -> Result<LLMGraphAnalysisResult, AppError> {
let response = self.openai_client.chat().create(request).await?;
debug!("Received LLM response: {:?}", response);
@@ -128,12 +130,12 @@ impl<'a> IngressAnalyzer<'a> {
.choices
.first()
.and_then(|choice| choice.message.content.as_ref())
.ok_or(ProcessingError::LLMParsingError(
"No content found in LLM response".into(),
.ok_or(AppError::LLMParsing(
"No content found in LLM response".to_string(),
))
.and_then(|content| {
serde_json::from_str(content).map_err(|e| {
ProcessingError::LLMParsingError(format!(
AppError::LLMParsing(format!(
"Failed to parse LLM response into analysis: {}",
e
))
src/ingress/analysis/types/llm_analysis_result.rs
+10 -13
View File
@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
use tokio::task;
use crate::{
error::ProcessingError,
error::AppError,
storage::types::{
knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
knowledge_relationship::KnowledgeRelationship,
@@ -49,13 +49,13 @@ impl LLMGraphAnalysisResult {
///
/// # Returns
///
/// * `Result<(Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>), ProcessingError>` - A tuple containing vectors of `KnowledgeEntity` and `KnowledgeRelationship`.
/// * `Result<(Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>), AppError>` - A tuple containing vectors of `KnowledgeEntity` and `KnowledgeRelationship`.
pub async fn to_database_entities(
&self,
source_id: &str,
user_id: &str,
openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
) -> Result<(Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>), ProcessingError> {
) -> Result<(Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>), AppError> {
// Create mapper and pre-assign IDs
let mapper = Arc::new(Mutex::new(self.create_mapper()?));
@@ -70,7 +70,7 @@ impl LLMGraphAnalysisResult {
Ok((entities, relationships))
}
fn create_mapper(&self) -> Result<GraphMapper, ProcessingError> {
fn create_mapper(&self) -> Result<GraphMapper, AppError> {
let mut mapper = GraphMapper::new();
// Pre-assign all IDs
@@ -87,7 +87,7 @@ impl LLMGraphAnalysisResult {
user_id: &str,
mapper: Arc<Mutex<GraphMapper>>,
openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
) -> Result<Vec<KnowledgeEntity>, ProcessingError> {
) -> Result<Vec<KnowledgeEntity>, AppError> {
let futures: Vec<_> = self
.knowledge_entities
.iter()
@@ -116,10 +116,10 @@ impl LLMGraphAnalysisResult {
fn process_relationships(
&self,
mapper: Arc<Mutex<GraphMapper>>,
) -> Result<Vec<KnowledgeRelationship>, ProcessingError> {
) -> Result<Vec<KnowledgeRelationship>, AppError> {
let mut mapper_guard = mapper
.lock()
.map_err(|_| ProcessingError::GraphProcessingError("Failed to lock mapper".into()))?;
.map_err(|_| AppError::GraphMapper("Failed to lock mapper".into()))?;
self.relationships
.iter()
.map(|rel| {
@@ -142,18 +142,15 @@ async fn create_single_entity(
user_id: &str,
mapper: Arc<Mutex<GraphMapper>>,
openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
) -> Result<KnowledgeEntity, ProcessingError> {
) -> Result<KnowledgeEntity, AppError> {
let assigned_id = {
let mapper = mapper
.lock()
.map_err(|_| ProcessingError::GraphProcessingError("Failed to lock mapper".into()))?;
.map_err(|_| AppError::GraphMapper("Failed to lock mapper".into()))?;
mapper
.get_id(&llm_entity.key)
.ok_or_else(|| {
ProcessingError::GraphProcessingError(format!(
"ID not found for key: {}",
llm_entity.key
))
AppError::GraphMapper(format!("ID not found for key: {}", llm_entity.key))
})?
.to_string()
};
src/ingress/content_processor.rs
+6 -6
View File
@@ -4,7 +4,7 @@ use text_splitter::TextSplitter;
use tracing::{debug, info};
use crate::{
error::ProcessingError,
error::AppError,
storage::{
db::{store_item, SurrealDbClient},
types::{
@@ -25,7 +25,7 @@ pub struct ContentProcessor {
}
impl ContentProcessor {
pub async fn new(app_config: &AppConfig) -> Result<Self, ProcessingError> {
pub async fn new(app_config: &AppConfig) -> Result<Self, AppError> {
Ok(Self {
db_client: SurrealDbClient::new(
&app_config.surrealdb_address,
@@ -39,7 +39,7 @@ impl ContentProcessor {
})
}
pub async fn process(&self, content: &TextContent) -> Result<(), ProcessingError> {
pub async fn process(&self, content: &TextContent) -> Result<(), AppError> {
// Store original content
store_item(&self.db_client, content.clone()).await?;
@@ -72,7 +72,7 @@ impl ContentProcessor {
async fn perform_semantic_analysis(
&self,
content: &TextContent,
) -> Result<LLMGraphAnalysisResult, ProcessingError> {
) -> Result<LLMGraphAnalysisResult, AppError> {
let analyser = IngressAnalyzer::new(&self.db_client, &self.openai_client);
analyser
.analyze_content(
@@ -88,7 +88,7 @@ impl ContentProcessor {
&self,
entities: Vec<KnowledgeEntity>,
relationships: Vec<KnowledgeRelationship>,
) -> Result<(), ProcessingError> {
) -> Result<(), AppError> {
for entity in &entities {
debug!("Storing entity: {:?}", entity);
store_item(&self.db_client, entity.clone()).await?;
@@ -107,7 +107,7 @@ impl ContentProcessor {
Ok(())
}
async fn store_vector_chunks(&self, content: &TextContent) -> Result<(), ProcessingError> {
async fn store_vector_chunks(&self, content: &TextContent) -> Result<(), AppError> {
let splitter = TextSplitter::new(500..2000);
let chunks = splitter.chunks(&content.text);
src/ingress/types/ingress_input.rs
+8 -34
View File
@@ -1,10 +1,12 @@
use super::ingress_object::IngressObject;
use crate::storage::{
db::{get_item, SurrealDbClient},
types::file_info::FileInfo,
use crate::{
error::AppError,
storage::{
db::{get_item, SurrealDbClient},
types::file_info::FileInfo,
},
};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::info;
use url::Url;
@@ -17,34 +19,6 @@ pub struct IngressInput {
pub files: Option<Vec<String>>,
}
/// Error types for processing ingress content.
#[derive(Error, Debug)]
pub enum IngressContentError {
#[error("IO error occurred: {0}")]
Io(#[from] std::io::Error),
#[error("UTF-8 conversion error: {0}")]
Utf8(#[from] std::string::FromUtf8Error),
#[error("SurrealDb error: {0}")]
SurrealDbError(#[from] surrealdb::Error),
#[error("MIME type detection failed for input: {0}")]
MimeDetection(String),
#[error("Unsupported MIME type: {0}")]
UnsupportedMime(String),
#[error("URL parse error: {0}")]
UrlParse(#[from] url::ParseError),
#[error("UUID parse error: {0}")]
UuidParse(#[from] uuid::Error),
#[error("Redis error: {0}")]
RedisError(String),
}
/// Function to create ingress objects from input.
///
/// # Arguments
@@ -57,7 +31,7 @@ pub async fn create_ingress_objects(
input: IngressInput,
db_client: &SurrealDbClient,
user_id: &str,
) -> Result<Vec<IngressObject>, IngressContentError> {
) -> Result<Vec<IngressObject>, AppError> {
// Initialize list
let mut object_list = Vec::new();
@@ -103,7 +77,7 @@ pub async fn create_ingress_objects(
// If no objects are constructed, we return Err
if object_list.is_empty() {
return Err(IngressContentError::MimeDetection(
return Err(AppError::NotFound(
"No valid content or files provided".into(),
));
}
src/ingress/types/ingress_object.rs
+10 -15
View File
@@ -1,8 +1,9 @@
use crate::storage::types::{file_info::FileInfo, text_content::TextContent};
use crate::{
error::AppError,
storage::types::{file_info::FileInfo, text_content::TextContent},
};
use serde::{Deserialize, Serialize};
use super::ingress_input::IngressContentError;
/// Knowledge object type, containing the content or reference to it, as well as metadata
#[derive(Debug, Serialize, Deserialize, Clone)]
pub enum IngressObject {
@@ -34,7 +35,7 @@ impl IngressObject {
///
/// # Returns
/// `TextContent` - An object containing a text representation of the object, could be a scraped URL, parsed PDF, etc.
pub async fn to_text_content(&self) -> Result<TextContent, IngressContentError> {
pub async fn to_text_content(&self) -> Result<TextContent, AppError> {
match self {
IngressObject::Url {
url,
@@ -82,12 +83,12 @@ impl IngressObject {
}
/// Fetches and extracts text from a URL.
async fn fetch_text_from_url(_url: &str) -> Result<String, IngressContentError> {
async fn fetch_text_from_url(_url: &str) -> Result<String, AppError> {
unimplemented!()
}
/// Extracts text from a file based on its MIME type.
async fn extract_text_from_file(file_info: &FileInfo) -> Result<String, IngressContentError> {
async fn extract_text_from_file(file_info: &FileInfo) -> Result<String, AppError> {
match file_info.mime_type.as_str() {
"text/plain" => {
// Read the file and return its content
@@ -101,15 +102,11 @@ impl IngressObject {
}
"application/pdf" => {
// TODO: Implement PDF text extraction using a crate like `pdf-extract` or `lopdf`
Err(IngressContentError::UnsupportedMime(
file_info.mime_type.clone(),
))
Err(AppError::NotFound(file_info.mime_type.clone()))
}
"image/png" | "image/jpeg" => {
// TODO: Implement OCR on image using a crate like `tesseract`
Err(IngressContentError::UnsupportedMime(
file_info.mime_type.clone(),
))
Err(AppError::NotFound(file_info.mime_type.clone()))
}
"application/octet-stream" => {
let content = tokio::fs::read_to_string(&file_info.path).await?;
@@ -120,9 +117,7 @@ impl IngressObject {
Ok(content)
}
// Handle other MIME types as needed
_ => Err(IngressContentError::UnsupportedMime(
file_info.mime_type.clone(),
)),
_ => Err(AppError::NotFound(file_info.mime_type.clone())),
}
}
}