diff --git a/CHANGELOG.md b/CHANGELOG.md index 7833b78..5454fc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,11 @@ # Changelog ## Unreleased - Fix: edge case where navigation back to a chat page could trigger a new response generation -- Security: Misc security fixes +- Fix: chat references now validate and render more reliably +- Fix: improved admin access checks for restricted routes +- Performance: faster chat sidebar loads from cached conversation archive data +- API: harmonized ingest endpoint naming and added configurable ingest safety limits +- Security: hardened query handling and ingestion logging to reduce injection and data exposure risk ## 1.0.1 (2026-02-11) - Shipped an S3 storage backend so content can be stored in object storage instead of local disk, with configuration support for S3 deployments. diff --git a/common/migrations/definitions/20260116_000000_add_user_theme_preference.json b/common/migrations/definitions/20260116_000000_add_user_theme_preference.json new file mode 100644 index 0000000..ca56cf6 --- /dev/null +++ b/common/migrations/definitions/20260116_000000_add_user_theme_preference.json @@ -0,0 +1 @@ +{"schemas":"--- original\n+++ modified\n@@ -28,6 +28,7 @@\n # Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations)\n DEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY\n+DEFINE INDEX IF NOT EXISTS conversation_user_updated_at_idx ON conversation FIELDS user_id, updated_at; # For sidebar conversation projection ORDER BY\n\n # Defines the schema for the 'file' table (used by FileInfo).\n\n","events":null} \ No newline at end of file diff --git a/common/schemas/conversation.surql b/common/schemas/conversation.surql index 04d838d..e4e7bbd 100644 --- a/common/schemas/conversation.surql +++ b/common/schemas/conversation.surql @@ -13,3 +13,4 @@ DEFINE FIELD IF NOT EXISTS title ON conversation TYPE string; # Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations) DEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id; DEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY +DEFINE INDEX IF NOT EXISTS conversation_user_updated_at_idx ON conversation FIELDS user_id, updated_at; # For sidebar conversation projection ORDER BY diff --git a/common/src/storage/types/conversation.rs b/common/src/storage/types/conversation.rs index 0b9b3c3..d3d640c 100644 --- a/common/src/storage/types/conversation.rs +++ b/common/src/storage/types/conversation.rs @@ -10,6 +10,54 @@ stored_object!(Conversation, "conversation", { title: String }); +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)] +pub struct SidebarConversation { + #[serde(deserialize_with = "deserialize_sidebar_id")] + pub id: String, + pub title: String, +} + +struct SidebarIdVisitor; + +impl<'de> serde::de::Visitor<'de> for SidebarIdVisitor { + type Value = String; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a string id or a SurrealDB Thing") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + Ok(value.to_string()) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(value) + } + + fn visit_map(self, map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let thing = ::deserialize( + serde::de::value::MapAccessDeserializer::new(map), + )?; + Ok(thing.id.to_raw()) + } +} + +fn deserialize_sidebar_id<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_any(SidebarIdVisitor) +} + impl Conversation { pub fn new(user_id: String, title: String) -> Self { let now = Utc::now(); @@ -75,6 +123,23 @@ impl Conversation { Ok(()) } + + pub async fn get_user_sidebar_conversations( + user_id: &str, + db: &SurrealDbClient, + ) -> Result, AppError> { + let conversations: Vec = db + .client + .query( + "SELECT id, title, updated_at FROM type::table($table_name) WHERE user_id = $user_id ORDER BY updated_at DESC", + ) + .bind(("table_name", Self::table_name())) + .bind(("user_id", user_id.to_string())) + .await? + .take(0)?; + + Ok(conversations) + } } #[cfg(test)] @@ -249,6 +314,96 @@ mod tests { } } + #[tokio::test] + async fn test_get_user_sidebar_conversations_filters_and_orders_by_updated_at_desc() { + let namespace = "test_ns"; + let database = &Uuid::new_v4().to_string(); + let db = SurrealDbClient::memory(namespace, database) + .await + .expect("Failed to start in-memory surrealdb"); + + let user_id = "sidebar_user"; + let other_user_id = "other_user"; + let base = Utc::now(); + + let mut oldest = Conversation::new(user_id.to_string(), "Oldest".to_string()); + oldest.updated_at = base - chrono::Duration::minutes(30); + + let mut newest = Conversation::new(user_id.to_string(), "Newest".to_string()); + newest.updated_at = base - chrono::Duration::minutes(5); + + let mut middle = Conversation::new(user_id.to_string(), "Middle".to_string()); + middle.updated_at = base - chrono::Duration::minutes(15); + + let mut other_user = Conversation::new(other_user_id.to_string(), "Other".to_string()); + other_user.updated_at = base; + + db.store_item(oldest.clone()) + .await + .expect("Failed to store oldest conversation"); + db.store_item(newest.clone()) + .await + .expect("Failed to store newest conversation"); + db.store_item(middle.clone()) + .await + .expect("Failed to store middle conversation"); + db.store_item(other_user) + .await + .expect("Failed to store other-user conversation"); + + let sidebar_items = Conversation::get_user_sidebar_conversations(user_id, &db) + .await + .expect("Failed to get sidebar conversations"); + + assert_eq!(sidebar_items.len(), 3); + assert_eq!(sidebar_items[0].id, newest.id); + assert_eq!(sidebar_items[0].title, "Newest"); + assert_eq!(sidebar_items[1].id, middle.id); + assert_eq!(sidebar_items[1].title, "Middle"); + assert_eq!(sidebar_items[2].id, oldest.id); + assert_eq!(sidebar_items[2].title, "Oldest"); + } + + #[tokio::test] + async fn test_sidebar_projection_reflects_patch_title_and_updated_at_reorder() { + let namespace = "test_ns"; + let database = &Uuid::new_v4().to_string(); + let db = SurrealDbClient::memory(namespace, database) + .await + .expect("Failed to start in-memory surrealdb"); + + let user_id = "sidebar_patch_user"; + let base = Utc::now(); + + let mut first = Conversation::new(user_id.to_string(), "First".to_string()); + first.updated_at = base - chrono::Duration::minutes(20); + + let mut second = Conversation::new(user_id.to_string(), "Second".to_string()); + second.updated_at = base - chrono::Duration::minutes(10); + + db.store_item(first.clone()) + .await + .expect("Failed to store first conversation"); + db.store_item(second.clone()) + .await + .expect("Failed to store second conversation"); + + let before_patch = Conversation::get_user_sidebar_conversations(user_id, &db) + .await + .expect("Failed to get sidebar conversations before patch"); + assert_eq!(before_patch[0].id, second.id); + + Conversation::patch_title(&first.id, user_id, "First (renamed)", &db) + .await + .expect("Failed to patch conversation title"); + + let after_patch = Conversation::get_user_sidebar_conversations(user_id, &db) + .await + .expect("Failed to get sidebar conversations after patch"); + assert_eq!(after_patch[0].id, first.id); + assert_eq!(after_patch[0].title, "First (renamed)"); + } + #[tokio::test] async fn test_get_complete_conversation_with_messages() { // Setup in-memory database for testing diff --git a/devenv.nix b/devenv.nix index 62b3a76..dca78f4 100644 --- a/devenv.nix +++ b/devenv.nix @@ -10,6 +10,7 @@ packages = [ pkgs.openssl pkgs.nodejs + pkgs.watchman pkgs.vscode-langservers-extracted pkgs.cargo-dist pkgs.cargo-xwin @@ -34,6 +35,6 @@ processes = { surreal_db.exec = "docker run --rm --pull always -p 8000:8000 --net=host --user $(id -u) -v $(pwd)/database:/database surrealdb/surrealdb:latest-dev start rocksdb:/database/database.db --user root_user --pass root_password"; server.exec = "cargo watch -x 'run --bin main'"; - tailwind.exec = "cd html-router && npm run tailwind"; + tailwind.exec = "tailwindcss --cwd html-router -i app.css -o assets/style.css --watch=always"; }; } diff --git a/html-router/package.json b/html-router/package.json index 406ba7d..29a9107 100644 --- a/html-router/package.json +++ b/html-router/package.json @@ -2,7 +2,7 @@ "name": "html-router", "version": "1.0.0", "scripts": { - "tailwind": "npx @tailwindcss/cli -i app.css -o assets/style.css -w -m" + "tailwind": "tailwindcss -i app.css -o assets/style.css --watch=always" }, "author": "", "license": "ISC", diff --git a/html-router/src/html_state.rs b/html-router/src/html_state.rs index e6e284e..96279fc 100644 --- a/html-router/src/html_state.rs +++ b/html-router/src/html_state.rs @@ -1,11 +1,14 @@ -use common::storage::types::conversation::Conversation; +use common::storage::types::conversation::SidebarConversation; use common::storage::{db::SurrealDbClient, store::StorageManager}; use common::utils::embedding::EmbeddingProvider; use common::utils::template_engine::{ProvidesTemplateEngine, TemplateEngine}; use common::{create_template_engine, storage::db::ProvidesDb, utils::config::AppConfig}; use retrieval_pipeline::{reranking::RerankerPool, RetrievalStrategy}; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; use std::time::{Duration, Instant}; use tokio::sync::RwLock; use tracing::debug; @@ -23,15 +26,18 @@ pub struct HtmlState { pub reranker_pool: Option>, pub embedding_provider: Arc, conversation_archive_cache: Arc>>, + conversation_archive_cache_writes: Arc, } #[derive(Clone)] struct ConversationArchiveCacheEntry { - conversations: Vec, + conversations: Vec, expires_at: Instant, } const CONVERSATION_ARCHIVE_CACHE_TTL: Duration = Duration::from_secs(30); +const CONVERSATION_ARCHIVE_CACHE_MAX_USERS: usize = 1024; +const CONVERSATION_ARCHIVE_CACHE_CLEANUP_WRITE_INTERVAL: usize = 64; impl HtmlState { pub async fn new_with_resources( @@ -58,6 +64,7 @@ impl HtmlState { reranker_pool, embedding_provider, conversation_archive_cache: Arc::new(RwLock::new(HashMap::new())), + conversation_archive_cache_writes: Arc::new(AtomicUsize::new(0)), }) } @@ -72,34 +79,82 @@ impl HtmlState { pub async fn get_cached_conversation_archive( &self, user_id: &str, - ) -> Option> { - let cache = self.conversation_archive_cache.read().await; - let entry = cache.get(user_id)?; - if entry.expires_at <= Instant::now() { - return None; + ) -> Option> { + let now = Instant::now(); + let should_evict_expired = { + let cache = self.conversation_archive_cache.read().await; + if let Some(entry) = cache.get(user_id) { + if entry.expires_at > now { + return Some(entry.conversations.clone()); + } + true + } else { + false + } + }; + + if should_evict_expired { + let mut cache = self.conversation_archive_cache.write().await; + cache.remove(user_id); } - Some(entry.conversations.clone()) + + None } pub async fn set_cached_conversation_archive( &self, user_id: &str, - conversations: Vec, + conversations: Vec, ) { + let now = Instant::now(); let mut cache = self.conversation_archive_cache.write().await; cache.insert( user_id.to_string(), ConversationArchiveCacheEntry { conversations, - expires_at: Instant::now() + CONVERSATION_ARCHIVE_CACHE_TTL, + expires_at: now + CONVERSATION_ARCHIVE_CACHE_TTL, }, ); + + let writes = self + .conversation_archive_cache_writes + .fetch_add(1, Ordering::Relaxed) + + 1; + if writes % CONVERSATION_ARCHIVE_CACHE_CLEANUP_WRITE_INTERVAL == 0 { + Self::purge_expired_entries(&mut cache, now); + } + + Self::enforce_cache_capacity(&mut cache); } pub async fn invalidate_conversation_archive_cache(&self, user_id: &str) { let mut cache = self.conversation_archive_cache.write().await; cache.remove(user_id); } + + fn purge_expired_entries( + cache: &mut HashMap, + now: Instant, + ) { + cache.retain(|_, entry| entry.expires_at > now); + } + + fn enforce_cache_capacity(cache: &mut HashMap) { + if cache.len() <= CONVERSATION_ARCHIVE_CACHE_MAX_USERS { + return; + } + + let overflow = cache.len() - CONVERSATION_ARCHIVE_CACHE_MAX_USERS; + let mut by_expiry: Vec<(String, Instant)> = cache + .iter() + .map(|(user_id, entry)| (user_id.clone(), entry.expires_at)) + .collect(); + by_expiry.sort_by_key(|(_, expires_at)| *expires_at); + + for (user_id, _) in by_expiry.into_iter().take(overflow) { + cache.remove(&user_id); + } + } } impl ProvidesDb for HtmlState { fn db(&self) -> &Arc { @@ -117,3 +172,87 @@ impl crate::middlewares::response_middleware::ProvidesHtmlState for HtmlState { self } } + +#[cfg(test)] +mod tests { + use super::*; + use common::{ + storage::types::conversation::SidebarConversation, + utils::{ + config::{AppConfig, StorageKind}, + embedding::EmbeddingProvider, + }, + }; + + async fn test_state() -> HtmlState { + let namespace = "test_ns"; + let database = &uuid::Uuid::new_v4().to_string(); + let db = Arc::new( + SurrealDbClient::memory(namespace, database) + .await + .expect("Failed to create in-memory DB"), + ); + + let session_store = Arc::new( + db.create_session_store() + .await + .expect("Failed to create session store"), + ); + + let mut config = AppConfig::default(); + config.storage = StorageKind::Memory; + + let storage = StorageManager::new(&config) + .await + .expect("Failed to create storage manager"); + + let embedding_provider = Arc::new( + EmbeddingProvider::new_hashed(8).expect("Failed to create embedding provider"), + ); + + HtmlState::new_with_resources( + db, + Arc::new(async_openai::Client::new()), + session_store, + storage, + config, + None, + embedding_provider, + None, + ) + .await + .expect("Failed to create HtmlState") + } + + #[tokio::test] + async fn test_expired_conversation_archive_entry_is_evicted_on_read() { + let state = test_state().await; + let user_id = "expired-user"; + + { + let mut cache = state.conversation_archive_cache.write().await; + cache.insert( + user_id.to_string(), + ConversationArchiveCacheEntry { + conversations: vec![SidebarConversation { + id: "conv-1".to_string(), + title: "A stale chat".to_string(), + }], + expires_at: Instant::now() - Duration::from_secs(1), + }, + ); + } + + let cached = state.get_cached_conversation_archive(user_id).await; + assert!( + cached.is_none(), + "Expired cache entry should not be returned" + ); + + let cache = state.conversation_archive_cache.read().await; + assert!( + !cache.contains_key(user_id), + "Expired cache entry should be evicted after read" + ); + } +} diff --git a/html-router/src/middlewares/response_middleware.rs b/html-router/src/middlewares/response_middleware.rs index 7269a9b..5ec5691 100644 --- a/html-router/src/middlewares/response_middleware.rs +++ b/html-router/src/middlewares/response_middleware.rs @@ -19,7 +19,7 @@ use tracing::error; use crate::{html_state::HtmlState, AuthSessionType}; use common::storage::types::{ - conversation::Conversation, + conversation::{Conversation, SidebarConversation}, user::{Theme, User}, }; @@ -141,7 +141,7 @@ struct ContextWrapper<'a> { initial_theme: &'a str, is_authenticated: bool, user: Option<&'a TemplateUser>, - conversation_archive: Vec, + conversation_archive: Vec, #[serde(flatten)] context: HashMap, } @@ -194,7 +194,7 @@ where { conversation_archive = cached_archive; } else if let Ok(archive) = - User::get_user_conversations(&user_id, &html_state.db).await + Conversation::get_user_sidebar_conversations(&user_id, &html_state.db).await { html_state .set_cached_conversation_archive(&user_id, archive.clone()) diff --git a/html-router/src/routes/index/handlers.rs b/html-router/src/routes/index/handlers.rs index afc79fe..d38312d 100644 --- a/html-router/src/routes/index/handlers.rs +++ b/html-router/src/routes/index/handlers.rs @@ -42,9 +42,8 @@ pub async fn index_handler( return Ok(TemplateResponse::redirect("/signin")); }; - let (text_contents, _conversation_archive, stats, active_jobs) = try_join!( + let (text_contents, stats, active_jobs) = try_join!( User::get_latest_text_contents(&user.id, &state.db), - User::get_user_conversations(&user.id, &state.db), User::get_dashboard_stats(&user.id, &state.db), User::get_unfinished_ingestion_tasks(&user.id, &state.db) )?; diff --git a/html-router/templates/chat/new_chat_first_response.html b/html-router/templates/chat/new_chat_first_response.html index f19b384..5a0b2b4 100644 --- a/html-router/templates/chat/new_chat_first_response.html +++ b/html-router/templates/chat/new_chat_first_response.html @@ -12,16 +12,34 @@ \ No newline at end of file + (function () { + const newChatStreamId = 'ai-stream-{{ user_message.id }}'; + + document.getElementById('chat-input').addEventListener('keydown', function (e) { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + htmx.trigger('#chat-form', 'submit'); + } + }); + // Clear textarea after successful submission + document.getElementById('chat-form').addEventListener('htmx:afterRequest', function (e) { + if (e.detail.successful) { // Check if the request was successful + document.getElementById('chat-input').value = ''; // Clear the textarea + } + }); + + const refreshSidebarAfterFirstResponse = function (e) { + const streamEl = document.getElementById(newChatStreamId); + if (!streamEl || e.target !== streamEl) return; + + htmx.ajax('GET', '/chat/sidebar', { + target: '.drawer-side', + swap: 'outerHTML' + }); + + document.body.removeEventListener('htmx:sseClose', refreshSidebarAfterFirstResponse); + }; + + document.body.addEventListener('htmx:sseClose', refreshSidebarAfterFirstResponse); + })(); +