starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-04-17 22:49:43 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

fix: own implementation of stream parser

Browse Source
This commit is contained in:
Per Stark
2025-04-10 08:23:55 +02:00
parent 435547de66
commit 233df1b79a
8 changed files with 359 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/src/utils/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod config; pub mod config;
pub mod embedding; pub mod embedding;
pub mod stream_parser;
pub mod template_engine; pub mod template_engine;

679
common/src/utils/stream_parser.rs
View File

@@ -1,506 +1,451 @@
// This code is based on the json-stream-rust library (https://github.com/json-stream/json-stream-rust)
// Original code is MIT licensed
// Modified to fix escape character handling in strings
use serde_json::{json, Value}; use serde_json::{json, Value};
// Modified ObjectStatus enum to track escape character state
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
enum ObjectStatus { enum ObjectStatus {
// We are ready to start a new object.
Ready, Ready,
// We are in the beginning of a string, likely because we just received an opening quote. StringQuoteOpen(bool),
StringQuoteOpen {
is_escaped: bool, // New field to track if the next character is escaped
},
// We just finished a string, likely because we just received a closing quote.
StringQuoteClose, StringQuoteClose,
// We are in the middle of a scalar value, likely because we just received a digit.
Scalar { Scalar {
value_so_far: Vec<char>, value_so_far: Vec<char>,
}, },
ScalarNumber { ScalarNumber {
value_so_far: Vec<char>, value_so_far: Vec<char>,
}, },
// We just started a property, likely because we just received an opening brace or a comma in case of an existing object.
StartProperty, StartProperty,
// We are in the beginning of a key, likely because we just received a quote. We need to store the key_so_far because
// unlike the value, we cannot add the key to the object until it is complete.
KeyQuoteOpen { KeyQuoteOpen {
key_so_far: Vec<char>, key_so_far: Vec<char>,
is_escaped: bool, // New field to track if the next character is escaped escaped: bool,
}, },
// We just finished a key, likely because we just received a closing quote.
KeyQuoteClose { KeyQuoteClose {
key: Vec<char>, key: Vec<char>,
}, },
// We just finished a key, likely because we just received a colon.
Colon { Colon {
key: Vec<char>, key: Vec<char>,
}, },
// We are in the beginning of a value, likely because we just received a quote.
ValueQuoteOpen { ValueQuoteOpen {
key: Vec<char>, key: Vec<char>,
is_escaped: bool, // New field to track if the next character is escaped escaped: bool,
// We don't need to store the valueSoFar because we can add the value to the object immediately.
}, },
ValueQuoteClose, ValueQuoteClose,
// We are taking any value that is not a string. For these case we just store
// each character until we reach a comma or a closing brace and then we pare
// and add the value to the object.
ValueScalar { ValueScalar {
key: Vec<char>, key: Vec<char>,
value_so_far: Vec<char>, value_so_far: Vec<char>,
}, },
// We just finished the object, likely because we just received a closing brace.
Closed, Closed,
} }
// Modified add_char_into_object function to handle escaped quotes
fn add_char_into_object( fn add_char_into_object(
object: &mut Value, object: &mut Value,
current_status: &mut ObjectStatus, current_status: &mut ObjectStatus,
current_char: char, current_char: char,
) -> Result<(), String> { ) -> Result<(), String> {
// First, match on the current status and character to determine what to do match (&*object, &*current_status, current_char) {
match (current_status.clone(), current_char) { // String escape handling
// ------ String handling with escaping ------ (&Value::String(_), &ObjectStatus::StringQuoteOpen(true), '"') => {
(ObjectStatus::StringQuoteOpen { is_escaped: true }, '"') => {
// Handle escaped quote in string
if let Value::String(str) = object { if let Value::String(str) = object {
str.push('"'); str.push('"');
} }
*current_status = ObjectStatus::StringQuoteOpen { is_escaped: false }; *current_status = ObjectStatus::StringQuoteOpen(false);
return Ok(());
} }
(ObjectStatus::StringQuoteOpen { is_escaped: false }, '"') => { (&Value::String(_), &ObjectStatus::StringQuoteOpen(false), '"') => {
// End of string
*current_status = ObjectStatus::StringQuoteClose; *current_status = ObjectStatus::StringQuoteClose;
return Ok(());
} }
(ObjectStatus::StringQuoteOpen { is_escaped: true }, char) => { (&Value::String(_), &ObjectStatus::StringQuoteOpen(true), c) => {
// Handle other escaped characters // Handle other escaped chars
if let Value::String(str) = object { if let Value::String(str) = object {
str.push('\\'); str.push('\\');
str.push(char); str.push(c);
} }
*current_status = ObjectStatus::StringQuoteOpen { is_escaped: false }; *current_status = ObjectStatus::StringQuoteOpen(false);
return Ok(());
} }
(ObjectStatus::StringQuoteOpen { is_escaped: false }, '\\') => { (&Value::String(_), &ObjectStatus::StringQuoteOpen(false), '\\') => {
// Set escape flag for next character *current_status = ObjectStatus::StringQuoteOpen(true);
*current_status = ObjectStatus::StringQuoteOpen { is_escaped: true };
return Ok(());
} }
(ObjectStatus::StringQuoteOpen { is_escaped: false }, char) => { (&Value::String(_), &ObjectStatus::StringQuoteOpen(false), c) => {
// Regular character in string
if let Value::String(str) = object { if let Value::String(str) = object {
str.push(char); str.push(c);
} }
return Ok(());
} }
// ------ Key handling with escaping ------ // Key escape handling
( (&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: true, .. }, '"') => {
ObjectStatus::KeyQuoteOpen { if let ObjectStatus::KeyQuoteOpen {
key_so_far, ref mut key_so_far, ..
is_escaped: true, } = current_status
}, {
'"', key_so_far.push('"');
) => { *current_status = ObjectStatus::KeyQuoteOpen {
// Handle escaped quote in key key_so_far: key_so_far.clone(),
let mut new_key = key_so_far; escaped: false,
new_key.push('"'); };
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far: new_key,
is_escaped: false,
};
return Ok(());
}
(
ObjectStatus::KeyQuoteOpen {
key_so_far,
is_escaped: false,
},
'"',
) => {
// End of key
if let Value::Object(obj) = object {
obj.insert(key_so_far.iter().collect::<String>(), Value::Null);
} }
*current_status = ObjectStatus::KeyQuoteClose { key: key_so_far };
return Ok(());
} }
( (&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: false, .. }, '"') => {
ObjectStatus::KeyQuoteOpen { if let ObjectStatus::KeyQuoteOpen { ref key_so_far, .. } = current_status {
key_so_far, let key = key_so_far.iter().collect::<String>();
is_escaped: true, if let Value::Object(obj) = object {
}, obj.insert(key.clone(), Value::Null);
char,
) => {
// Handle other escaped characters in key
let mut new_key = key_so_far;
new_key.push('\\');
new_key.push(char);
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far: new_key,
is_escaped: false,
};
return Ok(());
}
(
ObjectStatus::KeyQuoteOpen {
key_so_far,
is_escaped: false,
},
'\\',
) => {
// Set escape flag for next character in key
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far,
is_escaped: true,
};
return Ok(());
}
(
ObjectStatus::KeyQuoteOpen {
mut key_so_far,
is_escaped: false,
},
char,
) => {
// Regular character in key
key_so_far.push(char);
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far,
is_escaped: false,
};
return Ok(());
}
// ------ Value quote handling with escaping ------
(
ObjectStatus::ValueQuoteOpen {
key,
is_escaped: true,
},
'"',
) => {
// Handle escaped quote in value
if let Value::Object(obj) = object {
let key_string = key.iter().collect::<String>();
if let Some(Value::String(value)) = obj.get_mut(&key_string) {
value.push('"');
} }
*current_status = ObjectStatus::KeyQuoteClose {
key: key_so_far.clone(),
};
} }
*current_status = ObjectStatus::ValueQuoteOpen {
key,
is_escaped: false,
};
return Ok(());
} }
( (&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: true, .. }, c) => {
ObjectStatus::ValueQuoteOpen { if let ObjectStatus::KeyQuoteOpen {
is_escaped: false, .. ref mut key_so_far, ..
}, } = current_status
'"', {
) => { key_so_far.push('\\');
// End of value key_so_far.push(c);
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far: key_so_far.clone(),
escaped: false,
};
}
}
(&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: false, .. }, '\\') => {
if let ObjectStatus::KeyQuoteOpen { ref key_so_far, .. } = current_status {
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far: key_so_far.clone(),
escaped: true,
};
}
}
(&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: false, .. }, c) => {
if let ObjectStatus::KeyQuoteOpen {
ref mut key_so_far, ..
} = current_status
{
key_so_far.push(c);
}
}
// Value string escape handling
(&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: true, .. }, '"') => {
if let ObjectStatus::ValueQuoteOpen { ref key, .. } = current_status {
let key_str = key.iter().collect::<String>();
if let Value::Object(obj) = object {
if let Some(Value::String(value)) = obj.get_mut(&key_str) {
value.push('"');
}
}
*current_status = ObjectStatus::ValueQuoteOpen {
key: key.clone(),
escaped: false,
};
}
}
(&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: false, .. }, '"') => {
*current_status = ObjectStatus::ValueQuoteClose; *current_status = ObjectStatus::ValueQuoteClose;
return Ok(());
} }
( (&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: true, .. }, c) => {
ObjectStatus::ValueQuoteOpen { if let ObjectStatus::ValueQuoteOpen { ref key, .. } = current_status {
key, let key_str = key.iter().collect::<String>();
is_escaped: true, if let Value::Object(obj) = object {
}, if let Some(Value::String(value)) = obj.get_mut(&key_str) {
char, value.push('\\');
) => { value.push(c);
// Handle other escaped characters in value }
if let Value::Object(obj) = object { }
let key_string = key.iter().collect::<String>(); *current_status = ObjectStatus::ValueQuoteOpen {
if let Some(Value::String(value)) = obj.get_mut(&key_string) { key: key.clone(),
value.push('\\'); escaped: false,
value.push(char); };
}
}
(&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: false, .. }, '\\') => {
if let ObjectStatus::ValueQuoteOpen { ref key, .. } = current_status {
*current_status = ObjectStatus::ValueQuoteOpen {
key: key.clone(),
escaped: true,
};
}
}
(&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: false, .. }, c) => {
if let ObjectStatus::ValueQuoteOpen { ref key, .. } = current_status {
let key_str = key.iter().collect::<String>();
if let Value::Object(obj) = object {
if let Some(Value::String(value)) = obj.get_mut(&key_str) {
value.push(c);
} else {
return Err(format!("Invalid value type for key {}", key_str));
}
} }
} }
*current_status = ObjectStatus::ValueQuoteOpen {
key,
is_escaped: false,
};
return Ok(());
}
(
ObjectStatus::ValueQuoteOpen {
key,
is_escaped: false,
},
'\\',
) => {
// Set escape flag for next character in value
*current_status = ObjectStatus::ValueQuoteOpen {
key,
is_escaped: true,
};
return Ok(());
}
(
ObjectStatus::ValueQuoteOpen {
key,
is_escaped: false,
},
char,
) => {
// Regular character in value
if let Value::Object(obj) = object {
let key_string = key.iter().collect::<String>();
if let Some(Value::String(value)) = obj.get_mut(&key_string) {
value.push(char);
} else {
return Err(format!("Invalid value type for key {}", key_string));
}
}
return Ok(());
} }
// Now let's handle the rest of the cases using the original pattern // All other cases from the original implementation
_ => {} (&Value::Null, &ObjectStatus::Ready, '"') => {
} *object = json!("");
*current_status = ObjectStatus::StringQuoteOpen(false);
}
(&Value::Null, &ObjectStatus::Ready, '{') => {
*object = json!({});
*current_status = ObjectStatus::StartProperty;
}
// If we reach here, it means we didn't handle the character in the specific escape handling above,
// so we fall back to the original logic
match (object, current_status, current_char) {
(val @ Value::Null, sts @ ObjectStatus::Ready, '"') => {
*val = json!("");
*sts = ObjectStatus::StringQuoteOpen { is_escaped: false };
}
(val @ Value::Null, sts @ ObjectStatus::Ready, '{') => {
*val = json!({});
*sts = ObjectStatus::StartProperty;
}
// ------ true ------ // ------ true ------
(val @ Value::Null, sts @ ObjectStatus::Ready, 't') => { (&Value::Null, &ObjectStatus::Ready, 't') => {
*val = json!(true); *object = json!(true);
*sts = ObjectStatus::Scalar { *current_status = ObjectStatus::Scalar {
value_so_far: vec!['t'], value_so_far: vec!['t'],
}; };
} }
( (&Value::Bool(true), &ObjectStatus::Scalar { .. }, 'r') => {
Value::Bool(true), if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'r', {
) if *value_so_far == vec!['t'] => { if *value_so_far == vec!['t'] {
value_so_far.push('r'); value_so_far.push('r');
}
}
} }
( (&Value::Bool(true), &ObjectStatus::Scalar { .. }, 'u') => {
Value::Bool(true), if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'u', {
) if *value_so_far == vec!['t', 'r'] => { if *value_so_far == vec!['t', 'r'] {
value_so_far.push('u'); value_so_far.push('u');
}
}
} }
(Value::Bool(true), sts @ ObjectStatus::Scalar { .. }, 'e') => { (&Value::Bool(true), &ObjectStatus::Scalar { .. }, 'e') => {
*sts = ObjectStatus::Closed; *current_status = ObjectStatus::Closed;
} }
// ------ false ------ // ------ false ------
(val @ Value::Null, sts @ ObjectStatus::Ready, 'f') => { (&Value::Null, &ObjectStatus::Ready, 'f') => {
*val = json!(false); *object = json!(false);
*sts = ObjectStatus::Scalar { *current_status = ObjectStatus::Scalar {
value_so_far: vec!['f'], value_so_far: vec!['f'],
}; };
} }
( (&Value::Bool(false), &ObjectStatus::Scalar { .. }, 'a') => {
Value::Bool(false), if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'a', {
) if *value_so_far == vec!['f'] => { if *value_so_far == vec!['f'] {
value_so_far.push('a'); value_so_far.push('a');
}
}
} }
( (&Value::Bool(false), &ObjectStatus::Scalar { .. }, 'l') => {
Value::Bool(false), if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'l', {
) if *value_so_far == vec!['f', 'a'] => { if *value_so_far == vec!['f', 'a'] {
value_so_far.push('l'); value_so_far.push('l');
}
}
} }
( (&Value::Bool(false), &ObjectStatus::Scalar { .. }, 's') => {
Value::Bool(false), if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
's', {
) if *value_so_far == vec!['f', 'a', 'l'] => { if *value_so_far == vec!['f', 'a', 'l'] {
value_so_far.push('s'); value_so_far.push('s');
}
}
} }
(Value::Bool(false), sts @ ObjectStatus::Scalar { .. }, 'e') => { (&Value::Bool(false), &ObjectStatus::Scalar { .. }, 'e') => {
*sts = ObjectStatus::Closed; *current_status = ObjectStatus::Closed;
} }
// ------ null ------ // ------ null ------
(val @ Value::Null, sts @ ObjectStatus::Ready, 'n') => { (&Value::Null, &ObjectStatus::Ready, 'n') => {
*val = json!(null); *object = json!(null);
*sts = ObjectStatus::Scalar { *current_status = ObjectStatus::Scalar {
value_so_far: vec!['n'], value_so_far: vec!['n'],
}; };
} }
( (&Value::Null, &ObjectStatus::Scalar { .. }, 'u') => {
Value::Null, if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'u', {
) if *value_so_far == vec!['n'] => { if *value_so_far == vec!['n'] {
value_so_far.push('u'); value_so_far.push('u');
}
}
} }
( (&Value::Null, &ObjectStatus::Scalar { .. }, 'l') => {
Value::Null, if let ObjectStatus::Scalar {
ObjectStatus::Scalar {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'l', {
) if *value_so_far == vec!['n', 'u'] => { if *value_so_far == vec!['n', 'u'] {
value_so_far.push('l'); value_so_far.push('l');
} } else if *value_so_far == vec!['n', 'u', 'l'] {
(Value::Null, sts @ ObjectStatus::Scalar { .. }, 'l') => { // This is for the second 'l' in "null"
*sts = ObjectStatus::Closed; *current_status = ObjectStatus::Closed;
}
}
} }
// ------ number ------ // ------ number ------
(val @ Value::Null, sts @ ObjectStatus::Ready, c @ '0'..='9') => { (&Value::Null, &ObjectStatus::Ready, c @ '0'..='9') => {
*val = Value::Number(c.to_digit(10).unwrap().into()); *object = Value::Number(c.to_digit(10).unwrap().into());
*sts = ObjectStatus::ScalarNumber { *current_status = ObjectStatus::ScalarNumber {
value_so_far: vec![c], value_so_far: vec![c],
}; };
} }
(val @ Value::Null, sts @ ObjectStatus::Ready, '-') => { (&Value::Null, &ObjectStatus::Ready, '-') => {
*val = Value::Number(0.into()); *object = Value::Number(0.into());
*sts = ObjectStatus::ScalarNumber { *current_status = ObjectStatus::ScalarNumber {
value_so_far: vec!['-'], value_so_far: vec!['-'],
}; };
} }
( (&Value::Number(_), &ObjectStatus::ScalarNumber { .. }, c @ '0'..='9') => {
Value::Number(ref mut num), if let ObjectStatus::ScalarNumber {
ObjectStatus::ScalarNumber {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
c @ '0'..='9', {
) => { value_so_far.push(c);
value_so_far.push(c); // Parse based on whether it's a float or int
// if there are any . in the value so far, then we need to parse the number as a float if let Value::Number(ref mut num) = object {
if value_so_far.contains(&'.') { if value_so_far.contains(&'.') {
let parsed_number = value_so_far let parsed_number = value_so_far
.iter() .iter()
.collect::<String>() .collect::<String>()
.parse::<f64>() .parse::<f64>()
.unwrap(); .unwrap();
if let Some(json_number) = serde_json::Number::from_f64(parsed_number) { if let Some(json_number) = serde_json::Number::from_f64(parsed_number) {
*num = json_number; *num = json_number;
}
} else {
let parsed_number = value_so_far
.iter()
.collect::<String>()
.parse::<i64>()
.unwrap();
*num = parsed_number.into();
}
} }
} else {
let parsed_number = value_so_far
.iter()
.collect::<String>()
.parse::<i64>()
.unwrap();
*num = parsed_number.into();
} }
} }
( (&Value::Number(_), &ObjectStatus::ScalarNumber { .. }, '.') => {
Value::Number(_), if let ObjectStatus::ScalarNumber {
ObjectStatus::ScalarNumber {
ref mut value_so_far, ref mut value_so_far,
}, } = current_status
'.', {
) => { value_so_far.push('.');
value_so_far.push('.'); }
} }
(Value::Object(_obj), sts @ ObjectStatus::StartProperty, '"') => {
*sts = ObjectStatus::KeyQuoteOpen { // Object handling
(&Value::Object(_), &ObjectStatus::StartProperty, '"') => {
*current_status = ObjectStatus::KeyQuoteOpen {
key_so_far: vec![], key_so_far: vec![],
is_escaped: false, escaped: false,
}; };
} }
(Value::Object(_obj), sts @ ObjectStatus::KeyQuoteClose { .. }, ':') => { (&Value::Object(_), &ObjectStatus::KeyQuoteClose { .. }, ':') => {
if let ObjectStatus::KeyQuoteClose { key } = sts.clone() { if let ObjectStatus::KeyQuoteClose { ref key } = current_status {
*sts = ObjectStatus::Colon { key: key.clone() }; *current_status = ObjectStatus::Colon { key: key.clone() };
} }
} }
(Value::Object(_obj), ObjectStatus::Colon { .. }, ' ' | '\n') => {} (&Value::Object(_), &ObjectStatus::Colon { .. }, ' ' | '\n') => {}
(Value::Object(ref mut obj), sts @ ObjectStatus::Colon { .. }, '"') => { (&Value::Object(_), &ObjectStatus::Colon { .. }, '"') => {
if let ObjectStatus::Colon { key } = sts.clone() { if let ObjectStatus::Colon { ref key } = current_status {
*sts = ObjectStatus::ValueQuoteOpen { let key_str = key.iter().collect::<String>();
if let Value::Object(obj) = object {
obj.insert(key_str, json!(""));
}
*current_status = ObjectStatus::ValueQuoteOpen {
key: key.clone(), key: key.clone(),
is_escaped: false, escaped: false,
}; };
// create an empty string for the value
obj.insert(key.iter().collect::<String>().clone(), json!(""));
} }
} }
// ------ Add Scalar Value ------ // ------ Add Scalar Value ------
(Value::Object(_obj), sts @ ObjectStatus::Colon { .. }, char) => { (&Value::Object(_), &ObjectStatus::Colon { .. }, char) => {
if let ObjectStatus::Colon { key } = sts.clone() { if let ObjectStatus::Colon { ref key } = current_status {
*sts = ObjectStatus::ValueScalar { *current_status = ObjectStatus::ValueScalar {
key: key.clone(), key: key.clone(),
value_so_far: vec![char], value_so_far: vec![char],
}; };
} }
} }
(Value::Object(ref mut obj), sts @ ObjectStatus::ValueScalar { .. }, ',') => { (&Value::Object(_), &ObjectStatus::ValueScalar { .. }, ',') => {
if let ObjectStatus::ValueScalar { key, value_so_far } = sts.clone() { if let ObjectStatus::ValueScalar {
let key_string = key.iter().collect::<String>(); ref key,
let value_string = value_so_far.iter().collect::<String>(); ref value_so_far,
let value = match value_string.parse::<Value>() { } = current_status
Ok(value) => value, {
Err(e) => { let key_str = key.iter().collect::<String>();
return Err(format!("Invalid value for key {}: {}", key_string, e)); let value_str = value_so_far.iter().collect::<String>();
if let Value::Object(obj) = object {
match value_str.parse::<Value>() {
Ok(value) => {
obj.insert(key_str, value);
}
Err(e) => return Err(format!("Invalid value for key {}: {}", key_str, e)),
} }
}; }
obj.insert(key_string, value); *current_status = ObjectStatus::StartProperty;
*sts = ObjectStatus::StartProperty;
} }
} }
(Value::Object(ref mut obj), sts @ ObjectStatus::ValueScalar { .. }, '}') => { (&Value::Object(_), &ObjectStatus::ValueScalar { .. }, '}') => {
if let ObjectStatus::ValueScalar { key, value_so_far } = sts.clone() { if let ObjectStatus::ValueScalar {
let key_string = key.iter().collect::<String>(); ref key,
let value_string = value_so_far.iter().collect::<String>(); ref value_so_far,
let value = match value_string.parse::<Value>() { } = current_status
Ok(value) => value, {
Err(e) => { let key_str = key.iter().collect::<String>();
return Err(format!("Invalid value for key {}: {}", key_string, e)); let value_str = value_so_far.iter().collect::<String>();
if let Value::Object(obj) = object {
match value_str.parse::<Value>() {
Ok(value) => {
obj.insert(key_str, value);
}
Err(e) => return Err(format!("Invalid value for key {}: {}", key_str, e)),
} }
}; }
obj.insert(key_string, value); *current_status = ObjectStatus::Closed;
*sts = ObjectStatus::Closed;
} }
} }
( (&Value::Object(_), &ObjectStatus::ValueScalar { .. }, char) => {
Value::Object(_obj), if let ObjectStatus::ValueScalar {
ObjectStatus::ValueScalar {
key: _key,
ref mut value_so_far, ref mut value_so_far,
}, ..
char, } = current_status
) => { {
// push the character into the value so far value_so_far.push(char);
value_so_far.push(char); }
} }
// ------ Finished taking value ------ // ------ Finished taking value ------
(Value::Object(_obj), sts @ ObjectStatus::ValueQuoteClose, ',') => { (&Value::Object(_), &ObjectStatus::ValueQuoteClose, ',') => {
*sts = ObjectStatus::StartProperty; *current_status = ObjectStatus::StartProperty;
} }
(Value::Object(_obj), sts @ ObjectStatus::ValueQuoteClose, '}') => { (&Value::Object(_), &ObjectStatus::ValueQuoteClose, '}') => {
*sts = ObjectStatus::Closed; *current_status = ObjectStatus::Closed;
} }
// ------ white spaces ------ // ------ white spaces ------
(_, _, ' ' | '\n') => {} (_, _, ' ' | '\n') => {}
(_val, st, c) => {
return Err(format!("Invalid character {} status: {:?}", c, st)); (val, st, c) => {
return Err(format!(
"Invalid character {} status: {:?} value: {:?}",
c, st, val
));
} }
} }
Ok(()) Ok(())
} }
// The rest of the code remains the same
#[cfg(debug_assertions)] #[cfg(debug_assertions)]
pub fn parse_stream(json_string: &str) -> Result<Value, String> { pub fn parse_stream(json_string: &str) -> Result<Value, String> {
let mut out: Value = Value::Null; let mut out: Value = Value::Null;
@@ -543,11 +488,9 @@ impl JsonStreamParser {
current_status: ObjectStatus::Ready, current_status: ObjectStatus::Ready,
} }
} }
pub fn add_char(&mut self, current_char: char) -> Result<(), String> { pub fn add_char(&mut self, current_char: char) -> Result<(), String> {
add_char_into_object(&mut self.object, &mut self.current_status, current_char) add_char_into_object(&mut self.object, &mut self.current_status, current_char)
} }
pub fn get_result(&self) -> &Value { pub fn get_result(&self) -> &Value {
&self.object &self.object
} }
@@ -557,7 +500,9 @@ macro_rules! param_test {
($($name:ident: $string:expr, $value:expr)*) => { ($($name:ident: $string:expr, $value:expr)*) => {
$( $(
mod $name { mod $name {
#[allow(unused_imports)]
use super::{parse_stream, JsonStreamParser}; use super::{parse_stream, JsonStreamParser};
#[allow(unused_imports)]
use serde_json::{Value, json}; use serde_json::{Value, json};
#[test] #[test]

1
html-router/Cargo.toml
View File

@@ -21,7 +21,6 @@ axum_typed_multipart = "0.12.1"
futures = "0.3.31" futures = "0.3.31"
tempfile = "3.12.0" tempfile = "3.12.0"
async-stream = "0.3.6" async-stream = "0.3.6"
json-stream-parser = "0.1.4"
minijinja = { version = "2.5.0", features = ["loader", "multi_template"] } minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
minijinja-autoreload = "2.5.0" minijinja-autoreload = "2.5.0"
minijinja-embed = { version = "2.8.0" } minijinja-embed = { version = "2.8.0" }

18
html-router/src/routes/chat/message_response_stream.rs
View File

@@ -21,7 +21,6 @@ use futures::{
stream::{self, once}, stream::{self, once},
Stream, StreamExt, TryStreamExt, Stream, StreamExt, TryStreamExt,
}; };
use json_stream_parser::JsonStreamParser;
use minijinja::Value; use minijinja::Value;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::from_str; use serde_json::from_str;
@@ -29,14 +28,17 @@ use surrealdb::{engine::any::Any, Surreal};
use tokio::sync::{mpsc::channel, Mutex}; use tokio::sync::{mpsc::channel, Mutex};
use tracing::{debug, error}; use tracing::{debug, error};
use common::storage::{ use common::{
db::SurrealDbClient, storage::{
types::{ db::SurrealDbClient,
conversation::Conversation, types::{
message::{Message, MessageRole}, conversation::Conversation,
system_settings::SystemSettings, message::{Message, MessageRole},
user::User, system_settings::SystemSettings,
user::User,
},
}, },
utils::stream_parser::JsonStreamParser,
}; };
use crate::html_state::HtmlState; use crate::html_state::HtmlState;

33
html-router/templates/chat/streaming_response.html
View File

@@ -24,4 +24,37 @@
} }
} }
) )
// Add listener for after content is settled
document.body.addEventListener('htmx:afterSettle', function (evt) {
// Check if the settled element has our specific class
// evt.detail.target might be the container, elt is often the element *making* the request
// We need the element *receiving* the swap. Let's target specifically.
const messageId = "{{user_message.id}}"; // Get the ID from the template context
const targetBubble = document.querySelector(`.ai-message-content-${messageId}`);
// Ensure we have the marked library and the target exists
if (targetBubble && typeof marked !== 'undefined') {
// Get the raw text content (which includes previously streamed parts)
// Exclude the spinner if it's still somehow there, though it should be hidden.
let rawContent = '';
targetBubble.childNodes.forEach(node => {
if (node.nodeType === Node.TEXT_NODE) {
rawContent += node.textContent;
} else if (node.nodeType === Node.ELEMENT_NODE && !node.classList.contains('loading')) {
// In case HTMX wraps text in spans or something unexpected later
rawContent += node.textContent;
}
});
console.log(rawContent);
// Sanitize BEFORE inserting potentially harmful HTML from Markdown
// It's better to sanitize *after* rendering if using DOMPurify
targetBubble.innerHTML = marked.parse(rawContent);
// Optional: Sanitize with DOMPurify *after* parsing for security
// if (typeof DOMPurify !== 'undefined') {
// targetBubble.innerHTML = DOMPurify.sanitize(marked.parse(rawContent));
// } else {
// targetBubble.innerHTML = marked.parse(rawContent); // Use with caution if markdown source isn't trusted
// }
}
});
</script> </script>

1
html-router/templates/head_base.html
View File

@@ -19,6 +19,7 @@
<script src="/assets/htmx-ext-sse.js" defer></script> <script src="/assets/htmx-ext-sse.js" defer></script>
<script src="/assets/theme-toggle.js" defer></script> <script src="/assets/theme-toggle.js" defer></script>
<script src="/assets/toast.js" defer></script> <script src="/assets/toast.js" defer></script>
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
<!-- Icons --> <!-- Icons -->
<link rel="icon" href="/assets/icon/favicon.ico"> <link rel="icon" href="/assets/icon/favicon.ico">

2
main/src/main.rs
View File

@@ -38,7 +38,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let session_store = Arc::new(db.create_session_store().await?); let session_store = Arc::new(db.create_session_store().await?);
let openai_client = Arc::new(async_openai::Client::with_config( let openai_client = Arc::new(async_openai::Client::with_config(
OpenAIConfig::new().with_api_key(&config.openai_api_key), async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
)); ));
let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?; let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?;

2
main/src/server.rs
View File

@@ -36,7 +36,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let session_store = Arc::new(db.create_session_store().await?); let session_store = Arc::new(db.create_session_store().await?);
let openai_client = Arc::new(async_openai::Client::with_config( let openai_client = Arc::new(async_openai::Client::with_config(
OpenAIConfig::new().with_api_key(&config.openai_api_key), async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
)); ));
let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?; let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?;