refactor: better separation of dependencies to crates

node stuff to html crate only

ingestion-pipeline/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "ingestion-pipeline"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+# Workspace dependencies
+tokio = { workspace = true }
+serde = { workspace = true }
+axum = { workspace = true }
+tracing = { workspace = true }
+serde_json = { workspace = true }
+futures = { workspace = true }
+async-openai = { workspace = true }
+surrealdb = { workspace = true }
+
+tiktoken-rs = "0.6.0"
+reqwest = {version = "0.12.12", features = ["charset", "json"]}
+scraper = "0.22.0"
+chrono = { version = "0.4.39", features = ["serde"] }
+text-splitter = "0.18.1"
+uuid = { version = "1.10.0", features = ["v4", "serde"] }
+
+common = { path = "../common" }
+composite-retrieval = { path = "../composite-retrieval" }