ingestion-pipeline crated init, begun moving

2026-05-20 22:57:18 +02:00 · 2025-03-06 15:29:13 +01:00
parent ef1478547e
commit 1a641db503
21 changed files with 648 additions and 601 deletions
@@ -0,0 +1,21 @@
+[package]
+name = "ingestion-pipeline"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+# Workspace dependencies
+tokio = { workspace = true }
+serde = { workspace = true }
+axum = { workspace = true }
+tracing = { workspace = true }
+serde_json = { workspace = true }
+
+async-openai = "0.24.1"
+tiktoken-rs = "0.6.0"
+reqwest = {version = "0.12.12", features = ["charset", "json"]}
+scraper = "0.22.0"
+chrono = { version = "0.4.39", features = ["serde"] }
+text-splitter = "0.18.1"
+
+common = { path = "../common" }