fix: improved db schema strictness

fix

common/migrations/20250921_120004_fix_datetime_fields.surql

@@ -0,0 +1,115 @@
+-- Align timestamp fields with SurrealDB native datetime type.
+
+-- User timestamps
+DEFINE FIELD OVERWRITE created_at ON user FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON user FLEXIBLE;
+
+UPDATE user SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE user SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON user TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON user TYPE datetime;
+
+-- Text content timestamps
+DEFINE FIELD OVERWRITE created_at ON text_content FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON text_content FLEXIBLE;
+
+UPDATE text_content SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE text_content SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON text_content TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON text_content TYPE datetime;
+
+REBUILD INDEX text_content_created_at_idx ON text_content;
+
+-- Text chunk timestamps
+DEFINE FIELD OVERWRITE created_at ON text_chunk FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON text_chunk FLEXIBLE;
+
+UPDATE text_chunk SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE text_chunk SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON text_chunk TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON text_chunk TYPE datetime;
+
+-- Knowledge entity timestamps
+DEFINE FIELD OVERWRITE created_at ON knowledge_entity FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON knowledge_entity FLEXIBLE;
+
+UPDATE knowledge_entity SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE knowledge_entity SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON knowledge_entity TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON knowledge_entity TYPE datetime;
+
+REBUILD INDEX knowledge_entity_created_at_idx ON knowledge_entity;
+
+-- Conversation timestamps
+DEFINE FIELD OVERWRITE created_at ON conversation FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON conversation FLEXIBLE;
+
+UPDATE conversation SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE conversation SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON conversation TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON conversation TYPE datetime;
+
+REBUILD INDEX conversation_created_at_idx ON conversation;
+
+-- Message timestamps
+DEFINE FIELD OVERWRITE created_at ON message FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON message FLEXIBLE;
+
+UPDATE message SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE message SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON message TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON message TYPE datetime;
+
+REBUILD INDEX message_updated_at_idx ON message;
+
+-- Ingestion task timestamps
+DEFINE FIELD OVERWRITE created_at ON ingestion_task FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON ingestion_task FLEXIBLE;
+
+UPDATE ingestion_task SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE ingestion_task SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON ingestion_task TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON ingestion_task TYPE datetime;
+
+REBUILD INDEX idx_ingestion_task_created ON ingestion_task;
+
+-- File timestamps
+DEFINE FIELD OVERWRITE created_at ON file FLEXIBLE;
+DEFINE FIELD OVERWRITE updated_at ON file FLEXIBLE;
+
+UPDATE file SET created_at = type::datetime(created_at)
+WHERE type::is::string(created_at) AND created_at != "";
+
+UPDATE file SET updated_at = type::datetime(updated_at)
+WHERE type::is::string(updated_at) AND updated_at != "";
+
+DEFINE FIELD OVERWRITE created_at ON file TYPE datetime;
+DEFINE FIELD OVERWRITE updated_at ON file TYPE datetime;

common/migrations/definitions/20250921_120004_fix_datetime_fields.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -18,8 +18,8 @@\n DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;\n\n # Custom fields from the Conversation struct\n DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\n@@ -34,8 +34,8 @@\n DEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;\n\n # Custom fields from the FileInfo struct\n DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\n@@ -54,8 +54,8 @@\n DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;\n\n DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n@@ -71,8 +71,8 @@\n DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;\n\n # Custom fields from the KnowledgeEntity struct\n DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\n@@ -102,8 +102,8 @@\n DEFINE TABLE IF NOT EXISTS message SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime;\n\n # Custom fields from the Message struct\n DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string;\n@@ -167,8 +167,8 @@\n DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;\n\n # Custom fields from the TextChunk struct\n DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;\n@@ -191,8 +191,8 @@\n DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime;\n\n # Custom fields from the TextContent struct\n DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;\n@@ -215,8 +215,8 @@\n DEFINE TABLE IF NOT EXISTS user SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime;\n\n # Custom fields from the User struct\n DEFINE FIELD IF NOT EXISTS email ON user TYPE string;\n","events":null}

common/schemas/conversation.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;
 
 # Custom fields from the Conversation struct
 DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;

common/schemas/file.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS file SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;
 
 # Custom fields from the FileInfo struct
 DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;

common/schemas/ingestion_task.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;
 
 DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;
 DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;

common/schemas/knowledge_entity.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;
 
 # Custom fields from the KnowledgeEntity struct
 DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;

common/schemas/message.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS message SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime;
 
 # Custom fields from the Message struct
 DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string;

common/schemas/text_chunk.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;
 
 # Custom fields from the TextChunk struct
 DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;

common/schemas/text_content.surql

@@ -3,8 +3,8 @@
 DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime;
 
 # Custom fields from the TextContent struct
 DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;

common/schemas/user.surql

@@ -4,8 +4,8 @@
 DEFINE TABLE IF NOT EXISTS user SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime;
+DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime;
 
 # Custom fields from the User struct
 DEFINE FIELD IF NOT EXISTS email ON user TYPE string;

common/src/storage/types/conversation.rs

@@ -67,7 +67,10 @@ impl Conversation {
         let _updated: Option<Self> = db
             .update((Self::table_name(), id))
             .patch(PatchOp::replace("/title", new_title.to_string()))
-            .patch(PatchOp::replace("/updated_at", Utc::now()))
+            .patch(PatchOp::replace(
+                "/updated_at",
+                surrealdb::Datetime::from(Utc::now()),
+            ))
             .await?;
 
         Ok(())

common/src/storage/types/ingestion_task.rs

@@ -67,7 +67,7 @@ impl IngestionTask {
             .patch(PatchOp::replace("/status", status))
             .patch(PatchOp::replace(
                 "/updated_at",
-                surrealdb::sql::Datetime::default(),
+                surrealdb::Datetime::from(Utc::now()),
             ))
             .await?;
 

common/src/storage/types/knowledge_entity.rs

@@ -103,6 +103,8 @@ impl KnowledgeEntity {
         );
         let embedding = generate_embedding(ai_client, &embedding_input, db_client).await?;
 
+        let now = Utc::now();
+
         db_client
             .client
             .query(
@@ -117,7 +119,7 @@ impl KnowledgeEntity {
             .bind(("table", Self::table_name()))
             .bind(("id", id.to_string()))
             .bind(("name", name.to_string()))
-            .bind(("updated_at", Utc::now()))
+            .bind(("updated_at", surrealdb::Datetime::from(now)))
             .bind(("entity_type", entity_type.to_owned()))
             .bind(("embedding", embedding))
             .bind(("description", description.to_string()))

common/src/storage/types/text_content.rs

@@ -101,7 +101,10 @@ impl TextContent {
             .patch(PatchOp::replace("/context", context))
             .patch(PatchOp::replace("/category", category))
             .patch(PatchOp::replace("/text", text))
-            .patch(PatchOp::replace("/updated_at", now))
+            .patch(PatchOp::replace(
+                "/updated_at",
+                surrealdb::Datetime::from(now),
+            ))
             .await?;
 
         Ok(())

common/src/storage/types/user.rs

@@ -1,6 +1,7 @@
 use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
 use async_trait::async_trait;
 use axum_session_auth::Authentication;
+use chrono_tz::Tz;
 use surrealdb::{engine::any::Any, Surreal};
 use uuid::Uuid;
 
@@ -55,9 +56,6 @@ impl Authentication<User, String, Surreal<Any>> for User {
 }
 
 fn validate_timezone(input: &str) -> String {
-    use chrono_tz::Tz;
-
-    // Check if it's a valid IANA timezone identifier
     match input.parse::<Tz>() {
         Ok(_) => input.to_owned(),
         Err(_) => {
@@ -187,8 +185,8 @@ impl User {
             .bind(("id", id))
             .bind(("email", email))
             .bind(("password", password))
-            .bind(("created_at", now))
-            .bind(("updated_at", now))
+            .bind(("created_at", surrealdb::Datetime::from(now)))
+            .bind(("updated_at", surrealdb::Datetime::from(now)))
             .bind(("timezone", validated_tz))
             .await?
             .take(1)?;
@@ -980,4 +978,56 @@ mod tests {
         let most_recent = conversations.iter().max_by_key(|c| c.created_at).unwrap();
         assert_eq!(retrieved[0].id, most_recent.id);
     }
+
+    #[tokio::test]
+    async fn test_get_latest_text_contents_returns_last_five() {
+        let db = setup_test_db().await;
+        let user_id = "latest_text_user";
+
+        let mut inserted_ids = Vec::new();
+        let base_time = chrono::Utc::now() - chrono::Duration::minutes(60);
+
+        for i in 0..12 {
+            let mut item = TextContent::new(
+                format!("Text {}", i),
+                Some(format!("Context {}", i)),
+                "Category".to_string(),
+                None,
+                None,
+                user_id.to_string(),
+            );
+
+            let timestamp = base_time + chrono::Duration::minutes(i);
+            item.created_at = timestamp;
+            item.updated_at = timestamp;
+
+            db.store_item(item.clone())
+                .await
+                .expect("Failed to store text content");
+
+            inserted_ids.push(item.id.clone());
+        }
+
+        let latest = User::get_latest_text_contents(user_id, &db)
+            .await
+            .expect("Failed to fetch latest text contents");
+
+        assert_eq!(latest.len(), 5, "Expected exactly five items");
+
+        let mut expected_ids = inserted_ids[inserted_ids.len() - 5..].to_vec();
+        expected_ids.reverse();
+
+        let returned_ids: Vec<String> = latest.iter().map(|item| item.id.clone()).collect();
+        assert_eq!(
+            returned_ids, expected_ids,
+            "Latest items did not match expectation"
+        );
+
+        for window in latest.windows(2) {
+            assert!(
+                window[0].created_at >= window[1].created_at,
+                "Results are not ordered by created_at descending"
+            );
+        }
+    }
 }