feat: caching chat history & dto

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 ## Unreleased
 - Fix: edge case where navigation back to a chat page could trigger a new response generation
-- Security: Misc security fixes
+- Fix: chat references now validate and render more reliably
+- Fix: improved admin access checks for restricted routes
+- Performance: faster chat sidebar loads from cached conversation archive data
+- API: harmonized ingest endpoint naming and added configurable ingest safety limits
+- Security: hardened query handling and ingestion logging to reduce injection and data exposure risk
 
 ## 1.0.1 (2026-02-11)
 - Shipped an S3 storage backend so content can be stored in object storage instead of local disk, with configuration support for S3 deployments.

common/migrations/definitions/20260116_000000_add_user_theme_preference.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -28,6 +28,7 @@\n # Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations)\n DEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY\n+DEFINE INDEX IF NOT EXISTS conversation_user_updated_at_idx ON conversation FIELDS user_id, updated_at; # For sidebar conversation projection ORDER BY\n\n # Defines the schema for the 'file' table (used by FileInfo).\n\n","events":null}

common/schemas/conversation.surql

@@ -13,3 +13,4 @@ DEFINE FIELD IF NOT EXISTS title ON conversation TYPE string;
 # Add indexes based on query patterns (get_complete_conversation ownership check, get_user_conversations)
 DEFINE INDEX IF NOT EXISTS conversation_user_id_idx ON conversation FIELDS user_id;
 DEFINE INDEX IF NOT EXISTS conversation_created_at_idx ON conversation FIELDS created_at; # For get_user_conversations ORDER BY
+DEFINE INDEX IF NOT EXISTS conversation_user_updated_at_idx ON conversation FIELDS user_id, updated_at; # For sidebar conversation projection ORDER BY

common/src/storage/types/conversation.rs

@@ -10,6 +10,54 @@ stored_object!(Conversation, "conversation", {
     title: String
 });
 
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]
+pub struct SidebarConversation {
+    #[serde(deserialize_with = "deserialize_sidebar_id")]
+    pub id: String,
+    pub title: String,
+}
+
+struct SidebarIdVisitor;
+
+impl<'de> serde::de::Visitor<'de> for SidebarIdVisitor {
+    type Value = String;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        formatter.write_str("a string id or a SurrealDB Thing")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(value.to_string())
+    }
+
+    fn visit_string<E>(self, value: String) -> Result<Self::Value, E>
+    where
+        E: serde::de::Error,
+    {
+        Ok(value)
+    }
+
+    fn visit_map<A>(self, map: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::MapAccess<'de>,
+    {
+        let thing = <surrealdb::sql::Thing as serde::Deserialize>::deserialize(
+            serde::de::value::MapAccessDeserializer::new(map),
+        )?;
+        Ok(thing.id.to_raw())
+    }
+}
+
+fn deserialize_sidebar_id<'de, D>(deserializer: D) -> Result<String, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    deserializer.deserialize_any(SidebarIdVisitor)
+}
+
 impl Conversation {
     pub fn new(user_id: String, title: String) -> Self {
         let now = Utc::now();
@@ -75,6 +123,23 @@ impl Conversation {
 
         Ok(())
     }
+
+    pub async fn get_user_sidebar_conversations(
+        user_id: &str,
+        db: &SurrealDbClient,
+    ) -> Result<Vec<SidebarConversation>, AppError> {
+        let conversations: Vec<SidebarConversation> = db
+            .client
+            .query(
+                "SELECT id, title, updated_at FROM type::table($table_name) WHERE user_id = $user_id ORDER BY updated_at DESC",
+            )
+            .bind(("table_name", Self::table_name()))
+            .bind(("user_id", user_id.to_string()))
+            .await?
+            .take(0)?;
+
+        Ok(conversations)
+    }
 }
 
 #[cfg(test)]
@@ -249,6 +314,96 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn test_get_user_sidebar_conversations_filters_and_orders_by_updated_at_desc() {
+        let namespace = "test_ns";
+        let database = &Uuid::new_v4().to_string();
+        let db = SurrealDbClient::memory(namespace, database)
+            .await
+            .expect("Failed to start in-memory surrealdb");
+
+        let user_id = "sidebar_user";
+        let other_user_id = "other_user";
+        let base = Utc::now();
+
+        let mut oldest = Conversation::new(user_id.to_string(), "Oldest".to_string());
+        oldest.updated_at = base - chrono::Duration::minutes(30);
+
+        let mut newest = Conversation::new(user_id.to_string(), "Newest".to_string());
+        newest.updated_at = base - chrono::Duration::minutes(5);
+
+        let mut middle = Conversation::new(user_id.to_string(), "Middle".to_string());
+        middle.updated_at = base - chrono::Duration::minutes(15);
+
+        let mut other_user = Conversation::new(other_user_id.to_string(), "Other".to_string());
+        other_user.updated_at = base;
+
+        db.store_item(oldest.clone())
+            .await
+            .expect("Failed to store oldest conversation");
+        db.store_item(newest.clone())
+            .await
+            .expect("Failed to store newest conversation");
+        db.store_item(middle.clone())
+            .await
+            .expect("Failed to store middle conversation");
+        db.store_item(other_user)
+            .await
+            .expect("Failed to store other-user conversation");
+
+        let sidebar_items = Conversation::get_user_sidebar_conversations(user_id, &db)
+            .await
+            .expect("Failed to get sidebar conversations");
+
+        assert_eq!(sidebar_items.len(), 3);
+        assert_eq!(sidebar_items[0].id, newest.id);
+        assert_eq!(sidebar_items[0].title, "Newest");
+        assert_eq!(sidebar_items[1].id, middle.id);
+        assert_eq!(sidebar_items[1].title, "Middle");
+        assert_eq!(sidebar_items[2].id, oldest.id);
+        assert_eq!(sidebar_items[2].title, "Oldest");
+    }
+
+    #[tokio::test]
+    async fn test_sidebar_projection_reflects_patch_title_and_updated_at_reorder() {
+        let namespace = "test_ns";
+        let database = &Uuid::new_v4().to_string();
+        let db = SurrealDbClient::memory(namespace, database)
+            .await
+            .expect("Failed to start in-memory surrealdb");
+
+        let user_id = "sidebar_patch_user";
+        let base = Utc::now();
+
+        let mut first = Conversation::new(user_id.to_string(), "First".to_string());
+        first.updated_at = base - chrono::Duration::minutes(20);
+
+        let mut second = Conversation::new(user_id.to_string(), "Second".to_string());
+        second.updated_at = base - chrono::Duration::minutes(10);
+
+        db.store_item(first.clone())
+            .await
+            .expect("Failed to store first conversation");
+        db.store_item(second.clone())
+            .await
+            .expect("Failed to store second conversation");
+
+        let before_patch = Conversation::get_user_sidebar_conversations(user_id, &db)
+            .await
+            .expect("Failed to get sidebar conversations before patch");
+        assert_eq!(before_patch[0].id, second.id);
+
+        Conversation::patch_title(&first.id, user_id, "First (renamed)", &db)
+            .await
+            .expect("Failed to patch conversation title");
+
+        let after_patch = Conversation::get_user_sidebar_conversations(user_id, &db)
+            .await
+            .expect("Failed to get sidebar conversations after patch");
+        assert_eq!(after_patch[0].id, first.id);
+        assert_eq!(after_patch[0].title, "First (renamed)");
+    }
+
     #[tokio::test]
     async fn test_get_complete_conversation_with_messages() {
         // Setup in-memory database for testing

devenv.nix

@@ -10,6 +10,7 @@
   packages = [
     pkgs.openssl
     pkgs.nodejs
+    pkgs.watchman
     pkgs.vscode-langservers-extracted
     pkgs.cargo-dist
     pkgs.cargo-xwin
@@ -34,6 +35,6 @@
   processes = {
     surreal_db.exec = "docker run --rm --pull always -p 8000:8000 --net=host --user $(id -u) -v $(pwd)/database:/database surrealdb/surrealdb:latest-dev start rocksdb:/database/database.db --user root_user --pass root_password";
     server.exec = "cargo watch -x 'run --bin main'";
-    tailwind.exec = "cd html-router && npm run tailwind";
+    tailwind.exec = "tailwindcss --cwd html-router -i app.css -o assets/style.css --watch=always";
   };
 }

html-router/package.json

@@ -2,7 +2,7 @@
   "name": "html-router",
   "version": "1.0.0",
   "scripts": {
-    "tailwind": "npx @tailwindcss/cli -i app.css -o assets/style.css -w -m"
+    "tailwind": "tailwindcss -i app.css -o assets/style.css --watch=always"
   },
   "author": "",
   "license": "ISC",

html-router/src/html_state.rs

@@ -1,11 +1,14 @@
-use common::storage::types::conversation::Conversation;
+use common::storage::types::conversation::SidebarConversation;
 use common::storage::{db::SurrealDbClient, store::StorageManager};
 use common::utils::embedding::EmbeddingProvider;
 use common::utils::template_engine::{ProvidesTemplateEngine, TemplateEngine};
 use common::{create_template_engine, storage::db::ProvidesDb, utils::config::AppConfig};
 use retrieval_pipeline::{reranking::RerankerPool, RetrievalStrategy};
 use std::collections::HashMap;
-use std::sync::Arc;
+use std::sync::{
+    atomic::{AtomicUsize, Ordering},
+    Arc,
+};
 use std::time::{Duration, Instant};
 use tokio::sync::RwLock;
 use tracing::debug;
@@ -23,15 +26,18 @@ pub struct HtmlState {
     pub reranker_pool: Option<Arc<RerankerPool>>,
     pub embedding_provider: Arc<EmbeddingProvider>,
     conversation_archive_cache: Arc<RwLock<HashMap<String, ConversationArchiveCacheEntry>>>,
+    conversation_archive_cache_writes: Arc<AtomicUsize>,
 }
 
 #[derive(Clone)]
 struct ConversationArchiveCacheEntry {
-    conversations: Vec<Conversation>,
+    conversations: Vec<SidebarConversation>,
     expires_at: Instant,
 }
 
 const CONVERSATION_ARCHIVE_CACHE_TTL: Duration = Duration::from_secs(30);
+const CONVERSATION_ARCHIVE_CACHE_MAX_USERS: usize = 1024;
+const CONVERSATION_ARCHIVE_CACHE_CLEANUP_WRITE_INTERVAL: usize = 64;
 
 impl HtmlState {
     pub async fn new_with_resources(
@@ -58,6 +64,7 @@ impl HtmlState {
             reranker_pool,
             embedding_provider,
             conversation_archive_cache: Arc::new(RwLock::new(HashMap::new())),
+            conversation_archive_cache_writes: Arc::new(AtomicUsize::new(0)),
         })
     }
 
@@ -72,34 +79,82 @@ impl HtmlState {
     pub async fn get_cached_conversation_archive(
         &self,
         user_id: &str,
-    ) -> Option<Vec<Conversation>> {
-        let cache = self.conversation_archive_cache.read().await;
-        let entry = cache.get(user_id)?;
-        if entry.expires_at <= Instant::now() {
-            return None;
+    ) -> Option<Vec<SidebarConversation>> {
+        let now = Instant::now();
+        let should_evict_expired = {
+            let cache = self.conversation_archive_cache.read().await;
+            if let Some(entry) = cache.get(user_id) {
+                if entry.expires_at > now {
+                    return Some(entry.conversations.clone());
+                }
+                true
+            } else {
+                false
+            }
+        };
+
+        if should_evict_expired {
+            let mut cache = self.conversation_archive_cache.write().await;
+            cache.remove(user_id);
         }
-        Some(entry.conversations.clone())
+
+        None
     }
 
     pub async fn set_cached_conversation_archive(
         &self,
         user_id: &str,
-        conversations: Vec<Conversation>,
+        conversations: Vec<SidebarConversation>,
     ) {
+        let now = Instant::now();
         let mut cache = self.conversation_archive_cache.write().await;
         cache.insert(
             user_id.to_string(),
             ConversationArchiveCacheEntry {
                 conversations,
-                expires_at: Instant::now() + CONVERSATION_ARCHIVE_CACHE_TTL,
+                expires_at: now + CONVERSATION_ARCHIVE_CACHE_TTL,
             },
         );
+
+        let writes = self
+            .conversation_archive_cache_writes
+            .fetch_add(1, Ordering::Relaxed)
+            + 1;
+        if writes % CONVERSATION_ARCHIVE_CACHE_CLEANUP_WRITE_INTERVAL == 0 {
+            Self::purge_expired_entries(&mut cache, now);
+        }
+
+        Self::enforce_cache_capacity(&mut cache);
     }
 
     pub async fn invalidate_conversation_archive_cache(&self, user_id: &str) {
         let mut cache = self.conversation_archive_cache.write().await;
         cache.remove(user_id);
     }
+
+    fn purge_expired_entries(
+        cache: &mut HashMap<String, ConversationArchiveCacheEntry>,
+        now: Instant,
+    ) {
+        cache.retain(|_, entry| entry.expires_at > now);
+    }
+
+    fn enforce_cache_capacity(cache: &mut HashMap<String, ConversationArchiveCacheEntry>) {
+        if cache.len() <= CONVERSATION_ARCHIVE_CACHE_MAX_USERS {
+            return;
+        }
+
+        let overflow = cache.len() - CONVERSATION_ARCHIVE_CACHE_MAX_USERS;
+        let mut by_expiry: Vec<(String, Instant)> = cache
+            .iter()
+            .map(|(user_id, entry)| (user_id.clone(), entry.expires_at))
+            .collect();
+        by_expiry.sort_by_key(|(_, expires_at)| *expires_at);
+
+        for (user_id, _) in by_expiry.into_iter().take(overflow) {
+            cache.remove(&user_id);
+        }
+    }
 }
 impl ProvidesDb for HtmlState {
     fn db(&self) -> &Arc<SurrealDbClient> {
@@ -117,3 +172,87 @@ impl crate::middlewares::response_middleware::ProvidesHtmlState for HtmlState {
         self
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use common::{
+        storage::types::conversation::SidebarConversation,
+        utils::{
+            config::{AppConfig, StorageKind},
+            embedding::EmbeddingProvider,
+        },
+    };
+
+    async fn test_state() -> HtmlState {
+        let namespace = "test_ns";
+        let database = &uuid::Uuid::new_v4().to_string();
+        let db = Arc::new(
+            SurrealDbClient::memory(namespace, database)
+                .await
+                .expect("Failed to create in-memory DB"),
+        );
+
+        let session_store = Arc::new(
+            db.create_session_store()
+                .await
+                .expect("Failed to create session store"),
+        );
+
+        let mut config = AppConfig::default();
+        config.storage = StorageKind::Memory;
+
+        let storage = StorageManager::new(&config)
+            .await
+            .expect("Failed to create storage manager");
+
+        let embedding_provider = Arc::new(
+            EmbeddingProvider::new_hashed(8).expect("Failed to create embedding provider"),
+        );
+
+        HtmlState::new_with_resources(
+            db,
+            Arc::new(async_openai::Client::new()),
+            session_store,
+            storage,
+            config,
+            None,
+            embedding_provider,
+            None,
+        )
+        .await
+        .expect("Failed to create HtmlState")
+    }
+
+    #[tokio::test]
+    async fn test_expired_conversation_archive_entry_is_evicted_on_read() {
+        let state = test_state().await;
+        let user_id = "expired-user";
+
+        {
+            let mut cache = state.conversation_archive_cache.write().await;
+            cache.insert(
+                user_id.to_string(),
+                ConversationArchiveCacheEntry {
+                    conversations: vec![SidebarConversation {
+                        id: "conv-1".to_string(),
+                        title: "A stale chat".to_string(),
+                    }],
+                    expires_at: Instant::now() - Duration::from_secs(1),
+                },
+            );
+        }
+
+        let cached = state.get_cached_conversation_archive(user_id).await;
+        assert!(
+            cached.is_none(),
+            "Expired cache entry should not be returned"
+        );
+
+        let cache = state.conversation_archive_cache.read().await;
+        assert!(
+            !cache.contains_key(user_id),
+            "Expired cache entry should be evicted after read"
+        );
+    }
+}

html-router/src/middlewares/response_middleware.rs

@@ -19,7 +19,7 @@ use tracing::error;
 
 use crate::{html_state::HtmlState, AuthSessionType};
 use common::storage::types::{
-    conversation::Conversation,
+    conversation::{Conversation, SidebarConversation},
     user::{Theme, User},
 };
 
@@ -141,7 +141,7 @@ struct ContextWrapper<'a> {
     initial_theme: &'a str,
     is_authenticated: bool,
     user: Option<&'a TemplateUser>,
-    conversation_archive: Vec<Conversation>,
+    conversation_archive: Vec<SidebarConversation>,
     #[serde(flatten)]
     context: HashMap<String, Value>,
 }
@@ -194,7 +194,7 @@ where
                 {
                     conversation_archive = cached_archive;
                 } else if let Ok(archive) =
-                    User::get_user_conversations(&user_id, &html_state.db).await
+                    Conversation::get_user_sidebar_conversations(&user_id, &html_state.db).await
                 {
                     html_state
                         .set_cached_conversation_archive(&user_id, archive.clone())

html-router/src/routes/index/handlers.rs

@@ -42,9 +42,8 @@ pub async fn index_handler(
         return Ok(TemplateResponse::redirect("/signin"));
     };
 
-    let (text_contents, _conversation_archive, stats, active_jobs) = try_join!(
+    let (text_contents, stats, active_jobs) = try_join!(
         User::get_latest_text_contents(&user.id, &state.db),
-        User::get_user_conversations(&user.id, &state.db),
         User::get_dashboard_stats(&user.id, &state.db),
         User::get_unfinished_ingestion_tasks(&user.id, &state.db)
     )?;

html-router/templates/chat/new_chat_first_response.html

@@ -12,16 +12,34 @@
   </label>
 </form>
 <script>
-  document.getElementById('chat-input').addEventListener('keydown', function (e) {
-    if (e.key === 'Enter' && !e.shiftKey) {
-      e.preventDefault();
-      htmx.trigger('#chat-form', 'submit');
-    }
-  });
-  // Clear textarea after successful submission
-  document.getElementById('chat-form').addEventListener('htmx:afterRequest', function (e) {
-    if (e.detail.successful) { // Check if the request was successful
-      document.getElementById('chat-input').value = ''; // Clear the textarea
-    }
-  });
-</script>
+  (function () {
+    const newChatStreamId = 'ai-stream-{{ user_message.id }}';
+
+    document.getElementById('chat-input').addEventListener('keydown', function (e) {
+      if (e.key === 'Enter' && !e.shiftKey) {
+        e.preventDefault();
+        htmx.trigger('#chat-form', 'submit');
+      }
+    });
+    // Clear textarea after successful submission
+    document.getElementById('chat-form').addEventListener('htmx:afterRequest', function (e) {
+      if (e.detail.successful) { // Check if the request was successful
+        document.getElementById('chat-input').value = ''; // Clear the textarea
+      }
+    });
+
+    const refreshSidebarAfterFirstResponse = function (e) {
+      const streamEl = document.getElementById(newChatStreamId);
+      if (!streamEl || e.target !== streamEl) return;
+
+      htmx.ajax('GET', '/chat/sidebar', {
+        target: '.drawer-side',
+        swap: 'outerHTML'
+      });
+
+      document.body.removeEventListener('htmx:sseClose', refreshSidebarAfterFirstResponse);
+    };
+
+    document.body.addEventListener('htmx:sseClose', refreshSidebarAfterFirstResponse);
+  })();
+</script>