refactor: implemented state machines for retrieval pipeline, improved tracing

ingestion-pipeline/src/enricher.rs

@@ -9,10 +9,7 @@ use common::{
     error::AppError,
     storage::{db::SurrealDbClient, types::system_settings::SystemSettings},
 };
-use composite_retrieval::{
-    answer_retrieval::format_entities_json, retrieve_entities, RetrievedEntity,
-};
-use tracing::{debug, info};
+use composite_retrieval::{retrieve_entities, retrieved_entities_to_json, RetrievedEntity};
 
 use crate::{
     types::llm_enrichment_result::LLMEnrichmentResult,
@@ -42,11 +39,9 @@ impl IngestionEnricher {
         text: &str,
         user_id: &str,
     ) -> Result<LLMEnrichmentResult, AppError> {
-        info!("getting similar entitities");
         let similar_entities = self
             .find_similar_entities(category, context, text, user_id)
             .await?;
-        info!("got similar entitities");
         let llm_request = self
             .prepare_llm_request(category, context, text, &similar_entities)
             .await?;
@@ -60,9 +55,8 @@ impl IngestionEnricher {
         text: &str,
         user_id: &str,
     ) -> Result<Vec<RetrievedEntity>, AppError> {
-        let input_text = format!(
-            "content: {text}, category: {category}, user_context: {context:?}"
-        );
+        let input_text =
+            format!("content: {text}, category: {category}, user_context: {context:?}");
 
         retrieve_entities(&self.db_client, &self.openai_client, &input_text, user_id).await
     }
@@ -76,14 +70,12 @@ impl IngestionEnricher {
     ) -> Result<CreateChatCompletionRequest, AppError> {
         let settings = SystemSettings::get_current(&self.db_client).await?;
 
-        let entities_json = format_entities_json(similar_entities);
+        let entities_json = retrieved_entities_to_json(similar_entities);
 
         let user_message = format!(
             "Category:\n{category}\ncontext:\n{context:?}\nContent:\n{text}\nExisting KnowledgeEntities in database:\n{entities_json}"
         );
 
-        debug!("Prepared LLM request message: {}", user_message);
-
         let response_format = ResponseFormat::JsonSchema {
             json_schema: ResponseFormatJsonSchema {
                 description: Some("Structured analysis of the submitted content".into()),

ingestion-pipeline/src/pipeline.rs

@@ -2,7 +2,7 @@ use std::{sync::Arc, time::Instant};
 
 use text_splitter::TextSplitter;
 use tokio::time::{sleep, Duration};
-use tracing::{info, info_span, warn};
+use tracing::{debug, info, info_span, warn};
 
 use common::{
     error::AppError,
@@ -67,6 +67,34 @@ impl IngestionPipeline {
         )
         .await?;
 
+        let text_len = text_content.text.chars().count();
+        let preview: String = text_content.text.chars().take(120).collect();
+        let preview_clean = preview.replace("\n", " ");
+        let preview_len = preview_clean.chars().count();
+        let truncated = text_len > preview_len;
+        let context_len = text_content
+            .context
+            .as_ref()
+            .map(|c| c.chars().count())
+            .unwrap_or(0);
+        info!(
+            %task_id,
+            attempt,
+            user_id = %text_content.user_id,
+            category = %text_content.category,
+            text_chars = text_len,
+            context_chars = context_len,
+            attachments = text_content.file_info.is_some(),
+            "ingestion task input ready"
+        );
+        debug!(
+            %task_id,
+            attempt,
+            preview = %preview_clean,
+            preview_truncated = truncated,
+            "ingestion task input preview"
+        );
+
         match self.process(&text_content).await {
             Ok(()) => {
                 processing_task.mark_succeeded(&self.db).await?;

ingestion-pipeline/src/utils/pdf_ingestion.rs

@@ -132,9 +132,7 @@ async fn render_pdf_pages(file_path: &Path, pages: &[u32]) -> Result<Vec<Vec<u8>
     let mut captures = Vec::with_capacity(pages.len());
 
     for (idx, page) in pages.iter().enumerate() {
-        let target = format!(
-            "{file_url}#page={page}&toolbar=0&statusbar=0&zoom=page-fit"
-        );
+        let target = format!("{file_url}#page={page}&toolbar=0&statusbar=0&zoom=page-fit");
         tab.navigate_to(&target)
             .map_err(|err| AppError::Processing(format!("Failed to navigate to PDF page: {err}")))?
             .wait_until_navigated()
@@ -480,11 +478,7 @@ fn is_structural_line(line: &str) -> bool {
         || line.starts_with('~')
         || line.starts_with("| ")
         || line.starts_with("+-")
-        || lowered
-            .chars()
-            .next()
-            .is_some_and(|c| c.is_ascii_digit())
-            && lowered.contains('.')
+        || lowered.chars().next().is_some_and(|c| c.is_ascii_digit()) && lowered.contains('.')
 }
 
 fn debug_dump_directory() -> Option<PathBuf> {