ingestion-pipeline crated init, begun moving

Cargo.lock

@@ -1071,7 +1071,6 @@ dependencies = [
  "mockall",
  "plotly",
  "reqwest",
- "scraper",
  "serde",
  "serde_json",
  "sha2",
@@ -1079,7 +1078,6 @@ dependencies = [
  "tempfile",
  "text-splitter",
  "thiserror",
- "tiktoken-rs",
  "tokio",
  "tower-http",
  "tracing",
@@ -2429,6 +2427,24 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "ingestion-pipeline"
+version = "0.1.0"
+dependencies = [
+ "async-openai",
+ "axum",
+ "chrono",
+ "common",
+ "reqwest",
+ "scraper",
+ "serde",
+ "serde_json",
+ "text-splitter",
+ "tiktoken-rs",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "inotify"
 version = "0.10.2"
@@ -2734,6 +2750,7 @@ dependencies = [
  "config",
  "futures",
  "html-router",
+ "ingestion-pipeline",
  "json-stream-parser",
  "lettre",
  "mime",