fix: migrations

schemafull

common/migrations/20251120_000001_remove_legacy_hnsw_indexes.surql

@@ -0,0 +1,18 @@
+-- Remove HNSW indexes from base tables (now created at runtime on *_embedding tables)
+REMOVE INDEX IF EXISTS idx_embedding_entities ON knowledge_entity;
+REMOVE INDEX IF EXISTS idx_embedding_chunks ON text_chunk;
+
+-- Remove FTS indexes (now created at runtime via indexes.rs)
+REMOVE INDEX IF EXISTS text_content_fts_text_idx ON text_content;
+REMOVE INDEX IF EXISTS text_content_fts_category_idx ON text_content;
+REMOVE INDEX IF EXISTS text_content_fts_context_idx ON text_content;
+REMOVE INDEX IF EXISTS text_content_fts_file_name_idx ON text_content;
+REMOVE INDEX IF EXISTS text_content_fts_url_idx ON text_content;
+REMOVE INDEX IF EXISTS text_content_fts_url_title_idx ON text_content;
+REMOVE INDEX IF EXISTS knowledge_entity_fts_name_idx ON knowledge_entity;
+REMOVE INDEX IF EXISTS knowledge_entity_fts_description_idx ON knowledge_entity;
+REMOVE INDEX IF EXISTS text_chunk_fts_chunk_idx ON text_chunk;
+
+-- Remove legacy analyzers (recreated at runtime with updated configuration)
+REMOVE ANALYZER IF EXISTS app_default_fts_analyzer;
+REMOVE ANALYZER IF EXISTS app_en_fts_analyzer;

common/migrations/20251121_113122_migrate_embedding_data.surql

@@ -0,0 +1,23 @@
+-- Copy embeddings from base tables to dedicated tables
+-- This runs BEFORE the field removal migration
+
+FOR $chunk IN (SELECT * FROM text_chunk WHERE embedding != NONE AND array::len(embedding) > 0) {
+    CREATE text_chunk_embedding CONTENT {
+        chunk_id: $chunk.id,
+        embedding: $chunk.embedding,
+        user_id: $chunk.user_id,
+        source_id: $chunk.source_id,
+        created_at: $chunk.created_at,
+        updated_at: $chunk.updated_at
+    };
+};
+
+FOR $entity IN (SELECT * FROM knowledge_entity WHERE embedding != NONE AND array::len(embedding) > 0) {
+    CREATE knowledge_entity_embedding CONTENT {
+        entity_id: $entity.id,
+        embedding: $entity.embedding,
+        user_id: $entity.user_id,
+        created_at: $entity.created_at,
+        updated_at: $entity.updated_at
+    };
+};

common/migrations/20251231_enforce_schemafull.surql

@@ -0,0 +1,70 @@
+-- Enforce SCHEMAFULL on all tables and define missing fields
+
+-- 1. Define missing fields for ingestion_task (formerly job, but now ingestion_task)
+DEFINE TABLE OVERWRITE ingestion_task SCHEMAFULL;
+
+-- Core Fields
+DEFINE FIELD IF NOT EXISTS id ON ingestion_task TYPE record<ingestion_task>;
+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime DEFAULT time::now();
+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime DEFAULT time::now();
+DEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;
+
+-- State Machine Fields
+DEFINE FIELD IF NOT EXISTS state ON ingestion_task TYPE string ASSERT $value IN ['Pending', 'Reserved', 'Processing', 'Succeeded', 'Failed', 'Cancelled', 'DeadLetter'];
+DEFINE FIELD IF NOT EXISTS attempts ON ingestion_task TYPE int DEFAULT 0;
+DEFINE FIELD IF NOT EXISTS max_attempts ON ingestion_task TYPE int DEFAULT 3;
+DEFINE FIELD IF NOT EXISTS scheduled_at ON ingestion_task TYPE datetime DEFAULT time::now();
+DEFINE FIELD IF NOT EXISTS locked_at ON ingestion_task TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS lease_duration_secs ON ingestion_task TYPE int DEFAULT 300;
+DEFINE FIELD IF NOT EXISTS worker_id ON ingestion_task TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS error_code ON ingestion_task TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS error_message ON ingestion_task TYPE option<string>;
+DEFINE FIELD IF NOT EXISTS last_error_at ON ingestion_task TYPE option<datetime>;
+DEFINE FIELD IF NOT EXISTS priority ON ingestion_task TYPE int DEFAULT 0;
+
+-- Content Payload (IngestionPayload Enum)
+DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;
+DEFINE FIELD IF NOT EXISTS content.Url ON ingestion_task TYPE option<object>;
+DEFINE FIELD IF NOT EXISTS content.Text ON ingestion_task TYPE option<object>;
+DEFINE FIELD IF NOT EXISTS content.File ON ingestion_task TYPE option<object>;
+
+-- Content: Url Variant
+DEFINE FIELD IF NOT EXISTS content.Url.url ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Url.context ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Url.category ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Url.user_id ON ingestion_task TYPE string;
+
+-- Content: Text Variant
+DEFINE FIELD IF NOT EXISTS content.Text.text ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Text.context ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Text.category ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.Text.user_id ON ingestion_task TYPE string;
+
+-- Content: File Variant
+DEFINE FIELD IF NOT EXISTS content.File.context ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.category ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.user_id ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info ON ingestion_task TYPE object;
+
+-- Content: File.file_info (FileInfo Struct)
+DEFINE FIELD IF NOT EXISTS content.File.file_info.id ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.created_at ON ingestion_task TYPE datetime;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.updated_at ON ingestion_task TYPE datetime;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.sha256 ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.path ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.file_name ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.mime_type ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS content.File.file_info.user_id ON ingestion_task TYPE string;
+
+-- 2. Enforce SCHEMAFULL on all other tables
+DEFINE TABLE OVERWRITE analytics SCHEMAFULL;
+DEFINE TABLE OVERWRITE conversation SCHEMAFULL;
+DEFINE TABLE OVERWRITE file SCHEMAFULL;
+DEFINE TABLE OVERWRITE knowledge_entity SCHEMAFULL;
+DEFINE TABLE OVERWRITE message SCHEMAFULL;
+DEFINE TABLE OVERWRITE relates_to SCHEMAFULL;
+DEFINE TABLE OVERWRITE scratchpad SCHEMAFULL;
+DEFINE TABLE OVERWRITE system_settings SCHEMAFULL;
+DEFINE TABLE OVERWRITE text_chunk SCHEMAFULL;
+DEFINE TABLE OVERWRITE text_content SCHEMAFULL;
+DEFINE TABLE OVERWRITE user SCHEMAFULL;

common/migrations/definitions/20251122_151002_remove_legacy_embedding_fields.json

@@ -1 +0,0 @@
-{"schemas":"--- original\n+++ modified\n@@ -85,31 +85,30 @@\n\n DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity TYPE string;\n\n-# Indexes based on build_indexes and query patterns\n-# The INDEX definition correctly specifies the vector properties\n-# HNSW index now defined on knowledge_entity_embedding table for better memory usage  \n-# DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\n+-- Indexes based on build_indexes and query patterns\n+-- HNSW index now defined on knowledge_entity_embedding table for better memory usage\n+-- DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;\n\n-# Defines the schema for the 'knowledge_entity_embedding' table.\n-# Separate table to optimize HNSW index creation memory usage\n+-- Defines the schema for the 'knowledge_entity_embedding' table.\n+-- Separate table to optimize HNSW index creation memory usage\n\n DEFINE TABLE IF NOT EXISTS knowledge_entity_embedding SCHEMAFULL;\n\n-# Standard fields\n+-- Standard fields\n DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity_embedding TYPE datetime;\n DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity_embedding TYPE datetime;\n DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity_embedding TYPE string;\n\n-# Custom fields\n+-- Custom fields\n DEFINE FIELD IF NOT EXISTS entity_id ON knowledge_entity_embedding TYPE record<knowledge_entity>;\n DEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity_embedding TYPE array<float>;\n\n-# Indexes\n-# DEFINE INDEX IF NOT EXISTS idx_embedding_knowledge_entity_embedding ON knowledge_entity_embedding FIELDS embedding HNSW DIMENSION 1536;\n+-- Indexes\n+-- DEFINE INDEX IF NOT EXISTS idx_embedding_knowledge_entity_embedding ON knowledge_entity_embedding FIELDS embedding HNSW DIMENSION 1536;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_entity_id_idx ON knowledge_entity_embedding FIELDS entity_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_user_id_idx ON knowledge_entity_embedding FIELDS user_id;\n\n@@ -220,8 +219,8 @@\n DEFINE INDEX IF NOT EXISTS text_chunk_source_id_idx ON text_chunk FIELDS source_id;\n DEFINE INDEX IF NOT EXISTS text_chunk_user_id_idx ON text_chunk FIELDS user_id;\n\n-# Defines the schema for the 'text_chunk_embedding' table.\n-# Separate table to optimize HNSW index creation memory usage\n+-- Defines the schema for the 'text_chunk_embedding' table.\n+-- Separate table to optimize HNSW index creation memory usage\n\n DEFINE TABLE IF NOT EXISTS text_chunk_embedding SCHEMAFULL;\n\n@@ -235,8 +234,8 @@\n DEFINE FIELD IF NOT EXISTS chunk_id ON text_chunk_embedding TYPE record<text_chunk>;\n DEFINE FIELD IF NOT EXISTS embedding ON text_chunk_embedding TYPE array<float>;\n\n-# Indexes\n-# DEFINE INDEX IF NOT EXISTS idx_embedding_text_chunk_embedding ON text_chunk_embedding FIELDS embedding HNSW DIMENSION 1536;\n+-- Indexes\n+-- DEFINE INDEX IF NOT EXISTS idx_embedding_text_chunk_embedding ON text_chunk_embedding FIELDS embedding HNSW DIMENSION 1536;\n DEFINE INDEX IF NOT EXISTS text_chunk_embedding_chunk_id_idx ON text_chunk_embedding FIELDS chunk_id;\n DEFINE INDEX IF NOT EXISTS text_chunk_embedding_user_id_idx ON text_chunk_embedding FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS text_chunk_embedding_source_id_idx ON text_chunk_embedding FIELDS source_id;\n","events":null}

common/migrations/definitions/20251210_add_embedding_backend_to_system_settings.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -18,8 +18,8 @@\n DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;\n\n # Custom fields from the Conversation struct\n DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\n@@ -34,8 +34,8 @@\n DEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;\n\n # Custom fields from the FileInfo struct\n DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\n@@ -54,13 +54,10 @@\n DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;\n\n-# Custom fields from the IngestionTask struct\n-# IngestionPayload is complex, store as object\n DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n-# IngestionTaskStatus can hold data (InProgress), store as object\n DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n DEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;\n\n@@ -74,8 +71,8 @@\n DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;\n\n # Custom fields from the KnowledgeEntity struct\n DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\n@@ -86,27 +83,42 @@\n # metadata is Option<serde_json::Value>, store as object\n DEFINE FIELD IF NOT EXISTS metadata ON knowledge_entity TYPE option<object>;\n\n-# Define embedding as a standard array of floats for schema definition\n-DEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity TYPE array<float>;\n-# The specific vector nature is handled by the index definition below\n-\n DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity TYPE string;\n\n-# Indexes based on build_indexes and query patterns\n-# The INDEX definition correctly specifies the vector properties\n-DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\n-DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\n+-- Indexes based on build_indexes and query patterns\n+-- HNSW index now defined on knowledge_entity_embedding table for better memory usage\n+-- DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;\n+DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;\n-DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at; # For get_latest_knowledge_entities\n+DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;\n+\n+-- Defines the schema for the 'knowledge_entity_embedding' table.\n+-- Separate table to optimize HNSW index creation memory usage\n+\n+DEFINE TABLE IF NOT EXISTS knowledge_entity_embedding SCHEMAFULL;\n+\n+-- Standard fields\n+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity_embedding TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity_embedding TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity_embedding TYPE string;\n+\n+-- Custom fields\n+DEFINE FIELD IF NOT EXISTS entity_id ON knowledge_entity_embedding TYPE record<knowledge_entity>;\n+DEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity_embedding TYPE array<float>;\n\n+-- Indexes\n+-- DEFINE INDEX IF NOT EXISTS idx_embedding_knowledge_entity_embedding ON knowledge_entity_embedding FIELDS embedding HNSW DIMENSION 1536;\n+DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_entity_id_idx ON knowledge_entity_embedding FIELDS entity_id;\n+DEFINE INDEX IF NOT EXISTS knowledge_entity_embedding_user_id_idx ON knowledge_entity_embedding FIELDS user_id;\n+\n # Defines the schema for the 'message' table.\n\n DEFINE TABLE IF NOT EXISTS message SCHEMALESS;\n\n Line truncated

common/migrations/definitions/_initial.json

@@ -1 +1 @@
-{"schemas":"# Defines the schema for the 'analytics' table.\n\nDEFINE TABLE IF NOT EXISTS analytics SCHEMALESS;\n\n# Custom fields from the Analytics struct\nDEFINE FIELD IF NOT EXISTS page_loads ON analytics TYPE number;\nDEFINE FIELD IF NOT EXISTS visitors ON analytics TYPE number;\n\n# Defines authentication scope and access rules.\n# This mirrors the logic previously in SurrealDbClient::setup_auth\n\nDEFINE ACCESS IF NOT EXISTS account ON DATABASE TYPE RECORD\n    SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id) # Ensure user_id is provided if needed\n    SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );\n\n# Defines the schema for the 'conversation' table.\n\nDEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;\nDEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;\n\n# Custom fields from the Conversation struct\nDEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\nDEFINE FIELD IF NOT EXISTS title ON conversation TYPE string;\n\n# Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations)\nDEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY\n\n# Defines the schema for the 'file' table (used by FileInfo).\n\nDEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;\nDEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;\n\n# Custom fields from the FileInfo struct\nDEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS path ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS file_name ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS mime_type ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS user_id ON file TYPE string;\n\n# Indexes based on usage (get_by_sha, potentially user lookups)\n# Using UNIQUE based on the logic in FileInfo::new to prevent duplicates\nDEFINE INDEX IF NOT EXISTS file_sha256_idx ON file FIELDS sha256 UNIQUE;\nDEFINE INDEX IF NOT EXISTS file_user_id_idx ON file FIELDS user_id;\n\n# Defines the schema for the 'ingestion_task' table (used by IngestionTask).\n\nDEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;\nDEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;\n\nDEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\nDEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\nDEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;\n\n# Indexes explicitly defined in build_indexes and useful for get_unfinished_tasks\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_status ON ingestion_task FIELDS status;\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_user ON ingestion_task FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_created ON ingestion_task FIELDS created_at;\n\n# Defines the schema for the 'knowledge_entity' table.\n\nDEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;\nDEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;\n\n# Custom fields from the KnowledgeEntity struct\nDEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\nDEFINE FIELD IF NOT EXISTS name ON knowledge_entity TYPE string;\nDEFINE FIELD IF NOT EXISTS description ON knowledge_entity TYPE string;\n# KnowledgeEntityType is an enum, store as string\nDEFINE FIELD IF NOT EXISTS entity_type ON knowledge_entity TYPE string;\n# metadata is Option<serde_json::Value>, store as object\nDEFINE FIELD IF NOT EXISTS metadata ON knowledge_entity TYPE option<object>;\n\nDEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity TYPE string;\n\n# Indexes based on build_indexes and query patterns\n# The INDEX definition correctly specifies the vector properties\n# HNSW index now defined on knowledge_entity_embedding table for better memory usage  \n# DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;\n\n# Defines the schema for the 'knowledge_entity_embedding' table.\n# Separate table to optimize HNSW index creation memory usage\n\nDEFINE TABLE IF NOT EXISTS knowledge_eLine truncated
+{"schemas": "# Defines the schema for the 'analytics' table.\n\nDEFINE TABLE IF NOT EXISTS analytics SCHEMALESS;\n\n# Custom fields from the Analytics struct\nDEFINE FIELD IF NOT EXISTS page_loads ON analytics TYPE number;\nDEFINE FIELD IF NOT EXISTS visitors ON analytics TYPE number;\n\n# Defines authentication scope and access rules.\n# This mirrors the logic previously in SurrealDbClient::setup_auth\n\nDEFINE ACCESS IF NOT EXISTS account ON DATABASE TYPE RECORD\n    SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id) # Ensure user_id is provided if needed\n    SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );\n\n# Defines the schema for the 'conversation' table.\n\nDEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;\nDEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;\n\n# Custom fields from the Conversation struct\nDEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\nDEFINE FIELD IF NOT EXISTS title ON conversation TYPE string;\n\n# Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations)\nDEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY\n\n# Defines the schema for the 'file' table (used by FileInfo).\n\nDEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;\n\n# Custom fields from the FileInfo struct\nDEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS path ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS file_name ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS mime_type ON file TYPE string;\nDEFINE FIELD IF NOT EXISTS user_id ON file TYPE string;\n\n# Indexes based on usage (get_by_sha, potentially user lookups)\n# Using UNIQUE based on the logic in FileInfo::new to prevent duplicates\nDEFINE INDEX IF NOT EXISTS file_sha256_idx ON file FIELDS sha256 UNIQUE;\nDEFINE INDEX IF NOT EXISTS file_user_id_idx ON file FIELDS user_id;\n\n# Defines the schema for the 'ingestion_task' table (used by IngestionTask).\n\nDEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\nDEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n\n# Custom fields from the IngestionTask struct\n# IngestionPayload is complex, store as object\nDEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n# IngestionTaskStatus can hold data (InProgress), store as object\nDEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\nDEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;\n\n# Indexes explicitly defined in build_indexes and useful for get_unfinished_tasks\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_status ON ingestion_task FIELDS status;\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_user ON ingestion_task FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS idx_ingestion_task_created ON ingestion_task FIELDS created_at;\n\n# Defines the schema for the 'knowledge_entity' table.\n\nDEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n# Standard fields\nDEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;\nDEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;\n\n# Custom fields from the KnowledgeEntity struct\nDEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\nDEFINE FIELD IF NOT EXISTS name ON knowledge_entity TYPE string;\nDEFINE FIELD IF NOT EXISTS description ON knowledge_entity TYPE string;\n# KnowledgeEntityType is an enum, store as string\nDEFINE FIELD IF NOT EXISTS entity_type ON knowledge_entity TYPE string;\n# metadata is Option<serde_json::Value>, store as object\nDEFINE FIELD IF NOT EXISTS metadata ON knowledge_entity TYPE option<object>;\n\n# Define embedding as a standard array of floats for schema definition\nDEFINE FIELD IF NOT EXISTS embedding ON knowledge_entity TYPE array<float>;\n# The specific vector nature is handled by the index definition below\n\nDEFINE FIELD IF NOT EXISTS user_id ON knowledge_entity TYPE string;\n\n# Indexes based on build_indexes and query patterns\n# The INDEX definition correctly specifies the vector properties\nDEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS embedding HNSW DIMENSION 1536;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;\nDEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entitLine truncated

common/src/storage/types/text_content.rs

@@ -51,8 +51,11 @@ pub struct TextContentSearchResult {
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 pub struct UrlInfo {
+    #[serde(default)]
     pub url: String,
+    #[serde(default)]
     pub title: String,
+    #[serde(default)]
     pub image_id: String,
 }