chore: optimize ingest payloads and add parallel task batch store

Parse content before building file payloads to move shared metadata when
possible, add create_all_and_add_to_db for concurrent stores, and extend
tests for batch persistence and payload edge cases.

api-router/src/routes/ingest.rs

@@ -77,12 +77,7 @@ pub async fn ingest_data(
         user_id.clone(),
     )?;
 
-    let futures: Vec<_> = payloads
-        .into_iter()
-        .map(|object| IngestionTask::create_and_add_to_db(object, user_id.clone(), &state.db))
-        .collect();
-
-    try_join_all(futures).await?;
+    IngestionTask::create_all_and_add_to_db(payloads, &user_id, &state.db).await?;
 
     Ok((StatusCode::OK, Json(json!({ "status": "success" }))))
 }