benchmarks: ready for hybrid revised

2026-06-30 01:51:43 +02:00 · 2025-12-03 11:38:07 +01:00
parent 5e5053039a
commit c60efb2af7
22 changed files with 760 additions and 476 deletions
@@ -42,7 +42,9 @@ sha2 = "0.10.8"
 surrealdb-migrations = "2.2.2"
 surrealdb = { version = "2", features = ["kv-mem"] }
 tempfile = "3.12.0"
-text-splitter = "0.18.1"
+text-splitter = { version = "0.18.1", features = ["markdown", "tokenizers"] }
+tokenizers = { version = "0.20.4", features = ["http"] }
+unicode-normalization = "0.1.24"
 thiserror = "1.0.63"
 tokio-util = { version = "0.7.15", features = ["io"] }
 tokio = { version = "1", features = ["full"] }