fix: improve streaming message merging

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

sidecar/src/Eryx.AgentHost/Services/StreamingTextMerger.cs

@@ -38,6 +38,21 @@ internal static partial class StreamingTextMerger
             return incoming;
         }
 
+        return AppendWithNaturalBoundary(current, incoming);
+    }
+
+    private static string AppendWithNaturalBoundary(string current, string incoming)
+    {
+        if (ShouldInsertNewlineBoundary(current, incoming))
+        {
+            return current + "\n" + incoming;
+        }
+
+        if (ShouldInsertSpaceBoundary(current, incoming))
+        {
+            return current + " " + incoming;
+        }
+
         return current + incoming;
     }
 
@@ -62,8 +77,8 @@ internal static partial class StreamingTextMerger
             return false;
         }
 
-        HashSet<string> currentTokens = Tokenize(current);
-        HashSet<string> incomingTokens = Tokenize(incoming);
+        HashSet<string> currentTokens = Tokenize(current).ToHashSet(StringComparer.Ordinal);
+        HashSet<string> incomingTokens = Tokenize(incoming).ToHashSet(StringComparer.Ordinal);
         if (currentTokens.Count < 3 || incomingTokens.Count < 3)
         {
             return false;
@@ -73,15 +88,66 @@ internal static partial class StreamingTextMerger
         return shared / (double)Math.Min(currentTokens.Count, incomingTokens.Count) >= 0.5;
     }
 
-    private static HashSet<string> Tokenize(string value)
+    private static bool ShouldInsertNewlineBoundary(string current, string incoming)
+    {
+        if (current.EndsWith('\n'))
+        {
+            return false;
+        }
+
+        return MarkdownBlockPrefixRegex().IsMatch(incoming.TrimStart());
+    }
+
+    private static bool ShouldInsertSpaceBoundary(string current, string incoming)
+    {
+        char lastChar = current[^1];
+        char firstChar = incoming[0];
+
+        if (char.IsWhiteSpace(lastChar)
+            || char.IsWhiteSpace(firstChar)
+            || "([{/\"'`".Contains(lastChar))
+        {
+            return false;
+        }
+
+        if (ClosingPunctuationRegex().IsMatch(incoming))
+        {
+            return false;
+        }
+
+        if (MarkdownInlinePrefixRegex().IsMatch(incoming)
+            || char.IsUpper(firstChar)
+            || char.IsDigit(firstChar))
+        {
+            return true;
+        }
+
+        string[] currentTokens = Tokenize(current).ToArray();
+        string[] incomingTokens = Tokenize(incoming).ToArray();
+        string firstIncomingToken = incomingTokens.FirstOrDefault() ?? string.Empty;
+
+        return currentTokens.Length >= 2
+            && incomingTokens.Length >= 2
+            && firstIncomingToken.Length >= 2;
+    }
+
+    private static IEnumerable<string> Tokenize(string value)
     {
         return TokenRegex()
             .Matches(value.ToLowerInvariant())
             .Select(match => match.Value)
-            .Where(token => token.Length > 0)
-            .ToHashSet(StringComparer.Ordinal);
+            .Where(token => token.Length > 0);
     }
 
     [GeneratedRegex("[a-z0-9]+", RegexOptions.IgnoreCase)]
     private static partial Regex TokenRegex();
+
+    [GeneratedRegex(@"^[.,!?;:%)\]}]")]
+    private static partial Regex ClosingPunctuationRegex();
+
+    [GeneratedRegex(@"^[*_`~\[]")]
+    private static partial Regex MarkdownInlinePrefixRegex();
+
+    [GeneratedRegex(@"^(?:#{1,6}\s|[-*+]\s|\d+\.\s|>\s|```)", RegexOptions.Singleline)]
+    private static partial Regex MarkdownBlockPrefixRegex();
 }