From e8c67533f87ffe403a73de36fe68efe0dfa01c01 Mon Sep 17 00:00:00 2001
From: Per Stark <perstark.se@gmail.com>
Date: Tue, 22 Apr 2025 16:44:37 +0200
Subject: [PATCH] refactor: separation of json-stream-parser to own crate

---
 Cargo.lock                                     |  8 ++++++++
 Cargo.toml                                     |  2 +-
 common/src/utils/mod.rs                        |  1 -
 html-router/Cargo.toml                         |  1 +
 .../src/routes/chat/message_response_stream.rs | 18 ++++++++----------
 html-router/src/routes/knowledge/handlers.rs   |  2 +-
 json-stream-parser/Cargo.toml                  |  7 +++++++
 .../src/lib.rs                                 |  0
 8 files changed, 26 insertions(+), 13 deletions(-)
 create mode 100644 json-stream-parser/Cargo.toml
 rename common/src/utils/stream_parser.rs => json-stream-parser/src/lib.rs (100%)

diff --git a/Cargo.lock b/Cargo.lock
index a648df8..1d08fda 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2076,6 +2076,7 @@ dependencies = [
  "composite-retrieval",
  "futures",
  "include_dir",
+ "json-stream-parser",
  "minijinja",
  "minijinja-autoreload",
  "minijinja-contrib",
@@ -2601,6 +2602,13 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "json-stream-parser"
+version = "0.1.0"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "json5"
 version = "0.4.1"
diff --git a/Cargo.toml b/Cargo.toml
index f740acf..751f7e1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,7 @@ members = [
   "html-router",
   "ingestion-pipeline",
   "composite-retrieval"
-]
+, "json-stream-parser"]
 resolver = "2"
 
 [workspace.dependencies]
diff --git a/common/src/utils/mod.rs b/common/src/utils/mod.rs
index eed6261..10b3f33 100644
--- a/common/src/utils/mod.rs
+++ b/common/src/utils/mod.rs
@@ -1,4 +1,3 @@
 pub mod config;
 pub mod embedding;
-pub mod stream_parser;
 pub mod template_engine;
diff --git a/html-router/Cargo.toml b/html-router/Cargo.toml
index 25987f5..c65c629 100644
--- a/html-router/Cargo.toml
+++ b/html-router/Cargo.toml
@@ -34,6 +34,7 @@ include_dir = "0.7.4"
 
 common = { path = "../common" }
 composite-retrieval = { path = "../composite-retrieval" }
+json-stream-parser = { path = "../json-stream-parser" }
 
 [build-dependencies]
 minijinja-embed = { version = "2.8.0" }
diff --git a/html-router/src/routes/chat/message_response_stream.rs b/html-router/src/routes/chat/message_response_stream.rs
index dc1ea4e..a548e6c 100644
--- a/html-router/src/routes/chat/message_response_stream.rs
+++ b/html-router/src/routes/chat/message_response_stream.rs
@@ -21,6 +21,7 @@ use futures::{
     stream::{self, once},
     Stream, StreamExt, TryStreamExt,
 };
+use json_stream_parser::JsonStreamParser;
 use minijinja::Value;
 use serde::{Deserialize, Serialize};
 use serde_json::from_str;
@@ -28,17 +29,14 @@ use surrealdb::{engine::any::Any, Surreal};
 use tokio::sync::{mpsc::channel, Mutex};
 use tracing::{debug, error};
 
-use common::{
-    storage::{
-        db::SurrealDbClient,
-        types::{
-            conversation::Conversation,
-            message::{Message, MessageRole},
-            system_settings::SystemSettings,
-            user::User,
-        },
+use common::storage::{
+    db::SurrealDbClient,
+    types::{
+        conversation::Conversation,
+        message::{Message, MessageRole},
+        system_settings::SystemSettings,
+        user::User,
     },
-    utils::stream_parser::JsonStreamParser,
 };
 
 use crate::html_state::HtmlState;
diff --git a/html-router/src/routes/knowledge/handlers.rs b/html-router/src/routes/knowledge/handlers.rs
index 6d0725d..69b5135 100644
--- a/html-router/src/routes/knowledge/handlers.rs
+++ b/html-router/src/routes/knowledge/handlers.rs
@@ -258,7 +258,7 @@ fn get_plot_html(
     Ok(plot.to_html())
 }
 
-// Small utility to unzip tuple3 vectors from iterators (add this helper)
+// Small utility to unzip tuple3 vectors from iterators
 trait Unzip3<A, B, C> {
     fn unzip3(self) -> (Vec<A>, Vec<B>, Vec<C>);
 }
diff --git a/json-stream-parser/Cargo.toml b/json-stream-parser/Cargo.toml
new file mode 100644
index 0000000..69c630b
--- /dev/null
+++ b/json-stream-parser/Cargo.toml
@@ -0,0 +1,7 @@
+[package]
+name = "json-stream-parser"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+serde_json = { workspace = true }
diff --git a/common/src/utils/stream_parser.rs b/json-stream-parser/src/lib.rs
similarity index 100%
rename from common/src/utils/stream_parser.rs
rename to json-stream-parser/src/lib.rs