refactor: implemented state machine for ingestion pipeline, improved performance

changelog

additional moving around

moved files around a bit

Cargo.lock

@@ -2919,6 +2919,7 @@ name = "ingestion-pipeline"
 version = "0.1.0"
 dependencies = [
  "async-openai",
+ "async-trait",
  "axum",
  "axum_typed_multipart",
  "base64 0.22.1",
@@ -2933,6 +2934,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
+ "state-machines",
  "surrealdb",
  "tempfile",
  "text-splitter",