updated dependencies application wide

ingestion-pipeline/Cargo.toml

@@ -20,6 +20,7 @@ scraper = "0.22.0"
 chrono = { version = "0.4.39", features = ["serde"] }
 text-splitter = "0.18.1"
 uuid = { version = "1.10.0", features = ["v4", "serde"] }
+dom_smoothie = "0.10.0"
 
 common = { path = "../common" }
 composite-retrieval = { path = "../composite-retrieval" }

ingestion-pipeline/src/types/mod.rs

@@ -14,6 +14,7 @@ use common::{
         text_content::TextContent,
     },
 };
+use dom_smoothie::TextMode;
 use reqwest;
 use scraper::{Html, Selector};
 use std::fmt::Write;
@@ -125,6 +126,12 @@ async fn fetch_text_from_url(
         .replace("  ", " ");
 
     process_web_content(content, openai_client, db_client).await
+
+    // let config = dom_smoothie::Config {
+    //     text_mode: TextMode::Markdown,
+    //     ..Default::default()
+    // };
+    // panic!("YOU SHALL NOT PASS");
 }
 
 pub async fn process_web_content(