evals: v3, ebeddings at the side

additional indexes

common/schemas/knowledge_entity.surql

@@ -15,16 +15,12 @@ DEFINE FIELD IF NOT EXISTS entity_type ON knowledge_entity TYPE string;
 # metadata is Option<serde_json::Value>, store as object
 DEFINE FIELD IF NOT EXISTS metadata ON knowledge_entity TYPE option<object>;
 
-# Define embedding as a standard array of floats for schema definition
-DEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity TYPE array<float>;
-# The specific vector nature is handled by the index definition below
-
 DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity TYPE string;
 
-# Indexes based on build_indexes and query patterns
-# The INDEX definition correctly specifies the vector properties
-DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;
-DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;
+-- Indexes based on build_indexes and query patterns
+-- HNSW index now defined on knowledge_entity_embedding table for better memory usage
+-- DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;
 DEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;
+DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;
 DEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;
 DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;

common/schemas/knowledge_entity_embedding.surql

@@ -0,0 +1,18 @@
+-- Defines the schema for the 'knowledge_entity_embedding' table.
+-- Separate table to optimize HNSW index creation memory usage
+
+DEFINE TABLE IF NOT EXISTS knowledge_entity_embedding SCHEMAFULL;
+
+-- Standard fields
+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity_embedding TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity_embedding TYPE datetime;
+DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity_embedding TYPE string;
+
+-- Custom fields
+DEFINE FIELD IF NOT EXISTS entity_id ON knowledge_entity_embedding TYPE record<knowledge_entity>;
+DEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity_embedding TYPE array<float>;
+
+-- Indexes
+-- DEFINE INDEX IF NOT EXISTS idx_embedding_knowledge_entity_embedding ON knowledge_entity_embedding FIELDS embedding HNSW DIMENSION 1536;
+DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_entity_id_idx ON knowledge_entity_embedding FIELDS entity_id;
+DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_user_id_idx ON knowledge_entity_embedding FIELDS user_id;

common/schemas/text_chunk.surql

@@ -10,14 +10,8 @@ DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;
 DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;
 DEFINE FIELD IF NOT EXISTS chunk ON text_chunk TYPE string;
 
-# Define embedding as a standard array of floats for schema definition
-DEFINE FIELD IF NOT EXISTS embedding ON text_chunk TYPE array<float>;
-# The specific vector nature is handled by the index definition below
-
 DEFINE FIELD IF NOT EXISTS user_id ON text_chunk TYPE string;
 
 # Indexes based on build_indexes and query patterns (delete_by_source_id)
-# The INDEX definition correctly specifies the vector properties
-DEFINE INDEX IF NOT EXISTS idx_embedding_chunks ON text_chunk FIELDS embedding HNSW DIMENSION 1536;
 DEFINE INDEX IF NOT EXISTS text_chunk_source_id_idx ON text_chunk FIELDS source_id;
 DEFINE INDEX IF NOT EXISTS text_chunk_user_id_idx ON text_chunk FIELDS user_id;

common/schemas/text_chunk_embedding.surql

@@ -0,0 +1,20 @@
+-- Defines the schema for the 'text_chunk_embedding' table.
+-- Separate table to optimize HNSW index creation memory usage
+
+DEFINE TABLE IF NOT EXISTS text_chunk_embedding SCHEMAFULL;
+
+# Standard fields
+DEFINE FIELD IF NOT EXISTS created_at ON text_chunk_embedding TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk_embedding TYPE datetime;
+DEFINE FIELD IF NOT EXISTS user_id ON text_chunk_embedding TYPE string;
+DEFINE FIELD IF NOT EXISTS source_id ON text_chunk_embedding TYPE string;
+
+# Custom fields
+DEFINE FIELD IF NOT EXISTS chunk_id ON text_chunk_embedding TYPE record<text_chunk>;
+DEFINE FIELD IF NOT EXISTS embedding ON text_chunk_embedding TYPE array<float>;
+
+-- Indexes
+-- DEFINE INDEX IF NOT EXISTS idx_embedding_text_chunk_embedding ON text_chunk_embedding FIELDS embedding HNSW DIMENSION 1536;
+DEFINE INDEX IF NOT EXISTS text_chunk_embedding_chunk_id_idx ON text_chunk_embedding FIELDS chunk_id;
+DEFINE INDEX IF NOT EXISTS text_chunk_embedding_user_id_idx ON text_chunk_embedding FIELDS user_id;
+DEFINE INDEX IF NOT EXISTS text_chunk_embedding_source_id_idx ON text_chunk_embedding FIELDS source_id;