refactor: separation of json-stream-parser to own crate

Cargo.lock

@@ -2076,6 +2076,7 @@ dependencies = [
  "composite-retrieval",
  "futures",
  "include_dir",
+ "json-stream-parser",
  "minijinja",
  "minijinja-autoreload",
  "minijinja-contrib",
@@ -2601,6 +2602,13 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "json-stream-parser"
+version = "0.1.0"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "json5"
 version = "0.4.1"

Cargo.toml

@@ -6,7 +6,7 @@ members = [
   "html-router",
   "ingestion-pipeline",
   "composite-retrieval"
-]
+, "json-stream-parser"]
 resolver = "2"
 
 [workspace.dependencies]

common/src/utils/mod.rs

@@ -1,4 +1,3 @@
 pub mod config;
 pub mod embedding;
-pub mod stream_parser;
 pub mod template_engine;

html-router/Cargo.toml

@@ -34,6 +34,7 @@ include_dir = "0.7.4"
 
 common = { path = "../common" }
 composite-retrieval = { path = "../composite-retrieval" }
+json-stream-parser = { path = "../json-stream-parser" }
 
 [build-dependencies]
 minijinja-embed = { version = "2.8.0" }

html-router/src/routes/chat/message_response_stream.rs

@@ -21,6 +21,7 @@ use futures::{
     stream::{self, once},
     Stream, StreamExt, TryStreamExt,
 };
+use json_stream_parser::JsonStreamParser;
 use minijinja::Value;
 use serde::{Deserialize, Serialize};
 use serde_json::from_str;
@@ -28,17 +29,14 @@ use surrealdb::{engine::any::Any, Surreal};
 use tokio::sync::{mpsc::channel, Mutex};
 use tracing::{debug, error};
 
-use common::{
+use common::storage::{
-    storage::{
+    db::SurrealDbClient,
-        db::SurrealDbClient,
+    types::{
-        types::{
+        conversation::Conversation,
-            conversation::Conversation,
+        message::{Message, MessageRole},
-            message::{Message, MessageRole},
+        system_settings::SystemSettings,
-            system_settings::SystemSettings,
+        user::User,
-            user::User,
-        },
     },
-    utils::stream_parser::JsonStreamParser,
 };
 
 use crate::html_state::HtmlState;

html-router/src/routes/knowledge/handlers.rs

@@ -258,7 +258,7 @@ fn get_plot_html(
     Ok(plot.to_html())
 }
 
-// Small utility to unzip tuple3 vectors from iterators (add this helper)
+// Small utility to unzip tuple3 vectors from iterators
 trait Unzip3<A, B, C> {
     fn unzip3(self) -> (Vec<A>, Vec<B>, Vec<C>);
 }

json-stream-parser/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "json-stream-parser"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+serde_json = { workspace = true }