diff --git a/common/migrations/20250921_120004_fix_datetime_fields.surql b/common/migrations/20250921_120004_fix_datetime_fields.surql new file mode 100644 index 0000000..725a5e9 --- /dev/null +++ b/common/migrations/20250921_120004_fix_datetime_fields.surql @@ -0,0 +1,115 @@ +-- Align timestamp fields with SurrealDB native datetime type. + +-- User timestamps +DEFINE FIELD OVERWRITE created_at ON user FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON user FLEXIBLE; + +UPDATE user SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE user SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON user TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON user TYPE datetime; + +-- Text content timestamps +DEFINE FIELD OVERWRITE created_at ON text_content FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON text_content FLEXIBLE; + +UPDATE text_content SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE text_content SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON text_content TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON text_content TYPE datetime; + +REBUILD INDEX text_content_created_at_idx ON text_content; + +-- Text chunk timestamps +DEFINE FIELD OVERWRITE created_at ON text_chunk FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON text_chunk FLEXIBLE; + +UPDATE text_chunk SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE text_chunk SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON text_chunk TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON text_chunk TYPE datetime; + +-- Knowledge entity timestamps +DEFINE FIELD OVERWRITE created_at ON knowledge_entity FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON knowledge_entity FLEXIBLE; + +UPDATE knowledge_entity SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE knowledge_entity SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON knowledge_entity TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON knowledge_entity TYPE datetime; + +REBUILD INDEX knowledge_entity_created_at_idx ON knowledge_entity; + +-- Conversation timestamps +DEFINE FIELD OVERWRITE created_at ON conversation FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON conversation FLEXIBLE; + +UPDATE conversation SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE conversation SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON conversation TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON conversation TYPE datetime; + +REBUILD INDEX conversation_created_at_idx ON conversation; + +-- Message timestamps +DEFINE FIELD OVERWRITE created_at ON message FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON message FLEXIBLE; + +UPDATE message SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE message SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON message TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON message TYPE datetime; + +REBUILD INDEX message_updated_at_idx ON message; + +-- Ingestion task timestamps +DEFINE FIELD OVERWRITE created_at ON ingestion_task FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON ingestion_task FLEXIBLE; + +UPDATE ingestion_task SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE ingestion_task SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON ingestion_task TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON ingestion_task TYPE datetime; + +REBUILD INDEX idx_ingestion_task_created ON ingestion_task; + +-- File timestamps +DEFINE FIELD OVERWRITE created_at ON file FLEXIBLE; +DEFINE FIELD OVERWRITE updated_at ON file FLEXIBLE; + +UPDATE file SET created_at = type::datetime(created_at) +WHERE type::is::string(created_at) AND created_at != ""; + +UPDATE file SET updated_at = type::datetime(updated_at) +WHERE type::is::string(updated_at) AND updated_at != ""; + +DEFINE FIELD OVERWRITE created_at ON file TYPE datetime; +DEFINE FIELD OVERWRITE updated_at ON file TYPE datetime; diff --git a/common/migrations/definitions/20250921_120004_fix_datetime_fields.json b/common/migrations/definitions/20250921_120004_fix_datetime_fields.json new file mode 100644 index 0000000..d377e38 --- /dev/null +++ b/common/migrations/definitions/20250921_120004_fix_datetime_fields.json @@ -0,0 +1 @@ +{"schemas":"--- original\n+++ modified\n@@ -18,8 +18,8 @@\n DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;\n\n # Custom fields from the Conversation struct\n DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\n@@ -34,8 +34,8 @@\n DEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;\n\n # Custom fields from the FileInfo struct\n DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\n@@ -54,8 +54,8 @@\n DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;\n\n DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n@@ -71,8 +71,8 @@\n DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;\n\n # Custom fields from the KnowledgeEntity struct\n DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\n@@ -102,8 +102,8 @@\n DEFINE TABLE IF NOT EXISTS message SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime;\n\n # Custom fields from the Message struct\n DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string;\n@@ -167,8 +167,8 @@\n DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;\n\n # Custom fields from the TextChunk struct\n DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;\n@@ -191,8 +191,8 @@\n DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime;\n\n # Custom fields from the TextContent struct\n DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;\n@@ -215,8 +215,8 @@\n DEFINE TABLE IF NOT EXISTS user SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime;\n\n # Custom fields from the User struct\n DEFINE FIELD IF NOT EXISTS email ON user TYPE string;\n","events":null} \ No newline at end of file diff --git a/common/schemas/conversation.surql b/common/schemas/conversation.surql index 84e63b0..04d838d 100644 --- a/common/schemas/conversation.surql +++ b/common/schemas/conversation.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime; # Custom fields from the Conversation struct DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string; diff --git a/common/schemas/file.surql b/common/schemas/file.surql index 49fec18..08873b2 100644 --- a/common/schemas/file.surql +++ b/common/schemas/file.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS file SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime; # Custom fields from the FileInfo struct DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string; diff --git a/common/schemas/ingestion_task.surql b/common/schemas/ingestion_task.surql index f59d726..7defe5d 100644 --- a/common/schemas/ingestion_task.surql +++ b/common/schemas/ingestion_task.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime; DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object; DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object; diff --git a/common/schemas/knowledge_entity.surql b/common/schemas/knowledge_entity.surql index 0cf1eda..1fd95ab 100644 --- a/common/schemas/knowledge_entity.surql +++ b/common/schemas/knowledge_entity.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime; # Custom fields from the KnowledgeEntity struct DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string; diff --git a/common/schemas/message.surql b/common/schemas/message.surql index 4681f05..43486bf 100644 --- a/common/schemas/message.surql +++ b/common/schemas/message.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS message SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime; # Custom fields from the Message struct DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string; diff --git a/common/schemas/text_chunk.surql b/common/schemas/text_chunk.surql index 3bd9571..9d1fe16 100644 --- a/common/schemas/text_chunk.surql +++ b/common/schemas/text_chunk.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime; # Custom fields from the TextChunk struct DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string; diff --git a/common/schemas/text_content.surql b/common/schemas/text_content.surql index 133c7f7..e2f7b2c 100644 --- a/common/schemas/text_content.surql +++ b/common/schemas/text_content.surql @@ -3,8 +3,8 @@ DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime; # Custom fields from the TextContent struct DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string; diff --git a/common/schemas/user.surql b/common/schemas/user.surql index bfd1865..4a53f61 100644 --- a/common/schemas/user.surql +++ b/common/schemas/user.surql @@ -4,8 +4,8 @@ DEFINE TABLE IF NOT EXISTS user SCHEMALESS; # Standard fields -DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string; -DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string; +DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime; +DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime; # Custom fields from the User struct DEFINE FIELD IF NOT EXISTS email ON user TYPE string; diff --git a/common/src/storage/types/conversation.rs b/common/src/storage/types/conversation.rs index 6b1ab56..0b9b3c3 100644 --- a/common/src/storage/types/conversation.rs +++ b/common/src/storage/types/conversation.rs @@ -67,7 +67,10 @@ impl Conversation { let _updated: Option = db .update((Self::table_name(), id)) .patch(PatchOp::replace("/title", new_title.to_string())) - .patch(PatchOp::replace("/updated_at", Utc::now())) + .patch(PatchOp::replace( + "/updated_at", + surrealdb::Datetime::from(Utc::now()), + )) .await?; Ok(()) diff --git a/common/src/storage/types/ingestion_task.rs b/common/src/storage/types/ingestion_task.rs index 05ec924..22ef9de 100644 --- a/common/src/storage/types/ingestion_task.rs +++ b/common/src/storage/types/ingestion_task.rs @@ -67,7 +67,7 @@ impl IngestionTask { .patch(PatchOp::replace("/status", status)) .patch(PatchOp::replace( "/updated_at", - surrealdb::sql::Datetime::default(), + surrealdb::Datetime::from(Utc::now()), )) .await?; diff --git a/common/src/storage/types/knowledge_entity.rs b/common/src/storage/types/knowledge_entity.rs index cbdc50c..131ad6c 100644 --- a/common/src/storage/types/knowledge_entity.rs +++ b/common/src/storage/types/knowledge_entity.rs @@ -103,6 +103,8 @@ impl KnowledgeEntity { ); let embedding = generate_embedding(ai_client, &embedding_input, db_client).await?; + let now = Utc::now(); + db_client .client .query( @@ -117,7 +119,7 @@ impl KnowledgeEntity { .bind(("table", Self::table_name())) .bind(("id", id.to_string())) .bind(("name", name.to_string())) - .bind(("updated_at", Utc::now())) + .bind(("updated_at", surrealdb::Datetime::from(now))) .bind(("entity_type", entity_type.to_owned())) .bind(("embedding", embedding)) .bind(("description", description.to_string())) diff --git a/common/src/storage/types/text_content.rs b/common/src/storage/types/text_content.rs index 5321574..e8702b3 100644 --- a/common/src/storage/types/text_content.rs +++ b/common/src/storage/types/text_content.rs @@ -101,7 +101,10 @@ impl TextContent { .patch(PatchOp::replace("/context", context)) .patch(PatchOp::replace("/category", category)) .patch(PatchOp::replace("/text", text)) - .patch(PatchOp::replace("/updated_at", now)) + .patch(PatchOp::replace( + "/updated_at", + surrealdb::Datetime::from(now), + )) .await?; Ok(()) diff --git a/common/src/storage/types/user.rs b/common/src/storage/types/user.rs index 2b7e51d..5a00c0c 100644 --- a/common/src/storage/types/user.rs +++ b/common/src/storage/types/user.rs @@ -1,6 +1,7 @@ use crate::{error::AppError, storage::db::SurrealDbClient, stored_object}; use async_trait::async_trait; use axum_session_auth::Authentication; +use chrono_tz::Tz; use surrealdb::{engine::any::Any, Surreal}; use uuid::Uuid; @@ -55,9 +56,6 @@ impl Authentication> for User { } fn validate_timezone(input: &str) -> String { - use chrono_tz::Tz; - - // Check if it's a valid IANA timezone identifier match input.parse::() { Ok(_) => input.to_owned(), Err(_) => { @@ -187,8 +185,8 @@ impl User { .bind(("id", id)) .bind(("email", email)) .bind(("password", password)) - .bind(("created_at", now)) - .bind(("updated_at", now)) + .bind(("created_at", surrealdb::Datetime::from(now))) + .bind(("updated_at", surrealdb::Datetime::from(now))) .bind(("timezone", validated_tz)) .await? .take(1)?; @@ -980,4 +978,56 @@ mod tests { let most_recent = conversations.iter().max_by_key(|c| c.created_at).unwrap(); assert_eq!(retrieved[0].id, most_recent.id); } + + #[tokio::test] + async fn test_get_latest_text_contents_returns_last_five() { + let db = setup_test_db().await; + let user_id = "latest_text_user"; + + let mut inserted_ids = Vec::new(); + let base_time = chrono::Utc::now() - chrono::Duration::minutes(60); + + for i in 0..12 { + let mut item = TextContent::new( + format!("Text {}", i), + Some(format!("Context {}", i)), + "Category".to_string(), + None, + None, + user_id.to_string(), + ); + + let timestamp = base_time + chrono::Duration::minutes(i); + item.created_at = timestamp; + item.updated_at = timestamp; + + db.store_item(item.clone()) + .await + .expect("Failed to store text content"); + + inserted_ids.push(item.id.clone()); + } + + let latest = User::get_latest_text_contents(user_id, &db) + .await + .expect("Failed to fetch latest text contents"); + + assert_eq!(latest.len(), 5, "Expected exactly five items"); + + let mut expected_ids = inserted_ids[inserted_ids.len() - 5..].to_vec(); + expected_ids.reverse(); + + let returned_ids: Vec = latest.iter().map(|item| item.id.clone()).collect(); + assert_eq!( + returned_ids, expected_ids, + "Latest items did not match expectation" + ); + + for window in latest.windows(2) { + assert!( + window[0].created_at >= window[1].created_at, + "Results are not ordered by created_at descending" + ); + } + } }