diff --git a/Cargo.lock b/Cargo.lock index 0f30f5e..aeef8e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2475,6 +2475,7 @@ dependencies = [ "tower-http", "tower-serve-static", "tracing", + "url", ] [[package]] @@ -3183,17 +3184,25 @@ dependencies = [ "api-router", "async-openai", "axum", + "axum_session", + "axum_session_surreal", "common", + "cookie", "futures", + "headless_chrome", "html-router", "ingestion-pipeline", + "reqwest", "serde", "serde_json", + "serial_test", "surrealdb", + "tempfile", "thiserror 1.0.69", "tokio", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -5044,6 +5053,31 @@ dependencies = [ "syn 2.0.101", ] +[[package]] +name = "serial_test" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +dependencies = [ + "dashmap 5.5.3", + "futures", + "lazy_static", + "log", + "parking_lot", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "servo_arc" version = "0.4.0" diff --git a/html-router/Cargo.toml b/html-router/Cargo.toml index 8ac4576..ed6fc3f 100644 --- a/html-router/Cargo.toml +++ b/html-router/Cargo.toml @@ -31,6 +31,7 @@ chrono-tz = { workspace = true } tower-serve-static = { workspace = true } tokio-util = { workspace = true } chrono = { workspace = true } +url = { workspace = true } common = { path = "../common" } composite-retrieval = { path = "../composite-retrieval" } diff --git a/html-router/src/routes/content/handlers.rs b/html-router/src/routes/content/handlers.rs index fa14eac..f556e1e 100644 --- a/html-router/src/routes/content/handlers.rs +++ b/html-router/src/routes/content/handlers.rs @@ -17,8 +17,12 @@ use crate::{ auth_middleware::RequireUser, response_middleware::{HtmlError, TemplateResponse}, }, + utils::pagination::{paginate_items, Pagination}, utils::text_content_preview::truncate_text_contents, }; +use url::form_urlencoded; + +const CONTENTS_PER_PAGE: usize = 12; #[derive(Serialize)] pub struct ContentPageData { @@ -27,6 +31,8 @@ pub struct ContentPageData { categories: Vec, selected_category: Option, conversation_archive: Vec, + pagination: Pagination, + page_query: String, } #[derive(Serialize)] @@ -38,6 +44,7 @@ pub struct RecentTextContentData { #[derive(Deserialize)] pub struct FilterParams { category: Option, + page: Option, } pub async fn show_content_page( @@ -48,18 +55,31 @@ pub async fn show_content_page( HxBoosted(is_boosted): HxBoosted, ) -> Result { // Normalize empty strings to None - let has_category_param = params.category.is_some(); - let category_filter = params.category.as_deref().unwrap_or("").trim(); + let category_filter = params + .category + .as_ref() + .map(|c| c.trim()) + .filter(|c| !c.is_empty()); // load categories and filtered/all contents let categories = User::get_user_categories(&user.id, &state.db).await?; - let text_contents = if !category_filter.is_empty() { - User::get_text_contents_by_category(&user.id, category_filter, &state.db).await? - } else { - User::get_text_contents(&user.id, &state.db).await? + let full_contents = match category_filter { + Some(category) => { + User::get_text_contents_by_category(&user.id, category, &state.db).await? + } + None => User::get_text_contents(&user.id, &state.db).await?, }; - let text_contents = truncate_text_contents(text_contents); + let (page_contents, pagination) = paginate_items(full_contents, params.page, CONTENTS_PER_PAGE); + let text_contents = truncate_text_contents(page_contents); + + let page_query = category_filter + .map(|category| { + let mut serializer = form_urlencoded::Serializer::new(String::new()); + serializer.append_pair("category", category); + format!("&{}", serializer.finish()) + }) + .unwrap_or_default(); let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?; let data = ContentPageData { @@ -68,19 +88,19 @@ pub async fn show_content_page( categories, selected_category: params.category.clone(), conversation_archive, + pagination, + page_query, }; - if is_htmx && !is_boosted && has_category_param { - // If HTMX partial request with filter applied, return partial content list update - return Ok(TemplateResponse::new_partial( + if is_htmx && !is_boosted { + Ok(TemplateResponse::new_partial( "content/base.html", "main", data, - )); + )) + } else { + Ok(TemplateResponse::new_template("content/base.html", data)) } - - // Otherwise full page response including layout - Ok(TemplateResponse::new_template("content/base.html", data)) } pub async fn show_text_content_edit_form( @@ -132,7 +152,12 @@ pub async fn patch_text_content( )); } - let text_contents = truncate_text_contents(User::get_text_contents(&user.id, &state.db).await?); + let (page_contents, pagination) = paginate_items( + User::get_text_contents(&user.id, &state.db).await?, + Some(1), + CONTENTS_PER_PAGE, + ); + let text_contents = truncate_text_contents(page_contents); let categories = User::get_user_categories(&user.id, &state.db).await?; let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?; @@ -145,6 +170,8 @@ pub async fn patch_text_content( categories, selected_category: None, conversation_archive, + pagination, + page_query: String::new(), }, )) } @@ -170,7 +197,12 @@ pub async fn delete_text_content( state.db.delete_item::(&id).await?; // Get updated content, categories and return the refreshed list - let text_contents = truncate_text_contents(User::get_text_contents(&user.id, &state.db).await?); + let (page_contents, pagination) = paginate_items( + User::get_text_contents(&user.id, &state.db).await?, + Some(1), + CONTENTS_PER_PAGE, + ); + let text_contents = truncate_text_contents(page_contents); let categories = User::get_user_categories(&user.id, &state.db).await?; let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?; @@ -182,6 +214,8 @@ pub async fn delete_text_content( categories, selected_category: None, conversation_archive, + pagination, + page_query: String::new(), }, )) } diff --git a/html-router/src/routes/knowledge/handlers.rs b/html-router/src/routes/knowledge/handlers.rs index eb30227..1e50224 100644 --- a/html-router/src/routes/knowledge/handlers.rs +++ b/html-router/src/routes/knowledge/handlers.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use axum::{ extract::{Path, Query, State}, @@ -21,17 +21,23 @@ use crate::{ auth_middleware::RequireUser, response_middleware::{HtmlError, TemplateResponse}, }, + utils::pagination::{paginate_items, Pagination}, }; +use url::form_urlencoded; + +const KNOWLEDGE_ENTITIES_PER_PAGE: usize = 12; #[derive(Deserialize, Default)] pub struct FilterParams { entity_type: Option, content_category: Option, + page: Option, } #[derive(Serialize)] pub struct KnowledgeBaseData { entities: Vec, + visible_entities: Vec, relationships: Vec, user: User, entity_types: Vec, @@ -39,6 +45,8 @@ pub struct KnowledgeBaseData { selected_entity_type: Option, selected_content_category: Option, conversation_archive: Vec, + pagination: Pagination, + page_query: String, } pub async fn show_knowledge_page( @@ -67,11 +75,36 @@ pub async fn show_knowledge_page( }, }; + let (visible_entities, pagination) = + paginate_items(entities.clone(), params.page, KNOWLEDGE_ENTITIES_PER_PAGE); + + let page_query = { + let mut serializer = form_urlencoded::Serializer::new(String::new()); + if let Some(entity_type) = params.entity_type.as_deref() { + serializer.append_pair("entity_type", entity_type); + } + if let Some(content_category) = params.content_category.as_deref() { + serializer.append_pair("content_category", content_category); + } + let encoded = serializer.finish(); + if encoded.is_empty() { + String::new() + } else { + format!("&{}", encoded) + } + }; + let relationships = User::get_knowledge_relationships(&user.id, &state.db).await?; + let entity_id_set: HashSet = entities.iter().map(|e| e.id.clone()).collect(); + let relationships: Vec = relationships + .into_iter() + .filter(|rel| entity_id_set.contains(&rel.in_) && entity_id_set.contains(&rel.out)) + .collect(); let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?; let kb_data = KnowledgeBaseData { entities, + visible_entities, relationships, user, entity_types, @@ -79,20 +112,20 @@ pub async fn show_knowledge_page( selected_entity_type: params.entity_type.clone(), selected_content_category: params.content_category.clone(), conversation_archive, + pagination, + page_query, }; // Determine response type: // If it is an HTMX request but NOT a boosted navigation, send partial update (main block only) // Otherwise send full page including navbar/base for direct and boosted reloads if is_htmx && !is_boosted { - // Partial update (just main block) Ok(TemplateResponse::new_partial( "knowledge/base.html", "main", &kb_data, )) } else { - // Full page (includes navbar etc.) Ok(TemplateResponse::new_template( "knowledge/base.html", kb_data, @@ -145,8 +178,7 @@ pub async fn get_knowledge_graph_json( let relationships: Vec = User::get_knowledge_relationships(&user.id, &state.db).await?; - let entity_ids: std::collections::HashSet = - entities.iter().map(|e| e.id.clone()).collect(); + let entity_ids: HashSet = entities.iter().map(|e| e.id.clone()).collect(); let mut degree_count: HashMap = HashMap::new(); let mut links: Vec = Vec::new(); @@ -184,12 +216,24 @@ fn normalize_filter(input: Option) -> Option { if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("none") { None } else { - Some(trimmed.to_string()) + Some(trim_matching_quotes(trimmed).to_string()) } } } } +fn trim_matching_quotes(value: &str) -> &str { + let bytes = value.as_bytes(); + if bytes.len() >= 2 { + let first = bytes[0]; + let last = bytes[bytes.len() - 1]; + if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') { + return &value[1..value.len() - 1]; + } + } + value +} + pub async fn show_edit_knowledge_entity_form( State(state): State, RequireUser(user): RequireUser, @@ -231,12 +275,14 @@ pub struct PatchKnowledgeEntityParams { #[derive(Serialize)] pub struct EntityListData { - entities: Vec, + visible_entities: Vec, + pagination: Pagination, user: User, entity_types: Vec, content_categories: Vec, selected_entity_type: Option, selected_content_category: Option, + page_query: String, } pub async fn patch_knowledge_entity( @@ -261,7 +307,11 @@ pub async fn patch_knowledge_entity( .await?; // Get updated list of entities - let entities = User::get_knowledge_entities(&user.id, &state.db).await?; + let (visible_entities, pagination) = paginate_items( + User::get_knowledge_entities(&user.id, &state.db).await?, + Some(1), + KNOWLEDGE_ENTITIES_PER_PAGE, + ); // Get entity types let entity_types = User::get_entity_types(&user.id, &state.db).await?; @@ -273,12 +323,14 @@ pub async fn patch_knowledge_entity( Ok(TemplateResponse::new_template( "knowledge/entity_list.html", EntityListData { - entities, + visible_entities, + pagination, user, entity_types, content_categories, selected_entity_type: None, selected_content_category: None, + page_query: String::new(), }, )) } @@ -295,7 +347,11 @@ pub async fn delete_knowledge_entity( state.db.delete_item::(&id).await?; // Get updated list of entities - let entities = User::get_knowledge_entities(&user.id, &state.db).await?; + let (visible_entities, pagination) = paginate_items( + User::get_knowledge_entities(&user.id, &state.db).await?, + Some(1), + KNOWLEDGE_ENTITIES_PER_PAGE, + ); // Get entity types let entity_types = User::get_entity_types(&user.id, &state.db).await?; @@ -306,12 +362,14 @@ pub async fn delete_knowledge_entity( Ok(TemplateResponse::new_template( "knowledge/entity_list.html", EntityListData { - entities, + visible_entities, + pagination, user, entity_types, content_categories, selected_entity_type: None, selected_content_category: None, + page_query: String::new(), }, )) } diff --git a/html-router/src/utils/mod.rs b/html-router/src/utils/mod.rs index 5fba02a..4b487e6 100644 --- a/html-router/src/utils/mod.rs +++ b/html-router/src/utils/mod.rs @@ -1 +1,2 @@ +pub mod pagination; pub mod text_content_preview; diff --git a/html-router/src/utils/pagination.rs b/html-router/src/utils/pagination.rs new file mode 100644 index 0000000..7845f0c --- /dev/null +++ b/html-router/src/utils/pagination.rs @@ -0,0 +1,144 @@ +use serde::Serialize; + +/// Metadata describing a paginated collection. +#[derive(Debug, Clone, Serialize)] +pub struct Pagination { + pub current_page: usize, + pub per_page: usize, + pub total_items: usize, + pub total_pages: usize, + pub has_previous: bool, + pub has_next: bool, + pub previous_page: Option, + pub next_page: Option, + pub start_index: usize, + pub end_index: usize, +} + +impl Pagination { + pub fn new( + current_page: usize, + per_page: usize, + total_items: usize, + total_pages: usize, + page_len: usize, + ) -> Self { + let has_pages = total_pages > 0; + let has_previous = has_pages && current_page > 1; + let has_next = has_pages && current_page < total_pages; + let offset = if has_pages { + per_page.saturating_mul(current_page.saturating_sub(1)) + } else { + 0 + }; + let start_index = if page_len == 0 { 0 } else { offset + 1 }; + let end_index = if page_len == 0 { 0 } else { offset + page_len }; + + Self { + current_page, + per_page, + total_items, + total_pages, + has_previous, + has_next, + previous_page: if has_previous { + Some(current_page - 1) + } else { + None + }, + next_page: if has_next { + Some(current_page + 1) + } else { + None + }, + start_index, + end_index, + } + } +} + +/// Returns the items for the requested page along with pagination metadata. +pub fn paginate_items( + items: Vec, + requested_page: Option, + per_page: usize, +) -> (Vec, Pagination) { + let per_page = per_page.max(1); + let total_items = items.len(); + let total_pages = if total_items == 0 { + 0 + } else { + ((total_items - 1) / per_page) + 1 + }; + + let mut current_page = requested_page.unwrap_or(1); + if current_page == 0 { + current_page = 1; + } + if total_pages > 0 { + current_page = current_page.min(total_pages); + } else { + current_page = 1; + } + + let offset = if total_pages == 0 { + 0 + } else { + per_page.saturating_mul(current_page - 1) + }; + + let page_items: Vec = items.into_iter().skip(offset).take(per_page).collect(); + let page_len = page_items.len(); + let pagination = Pagination::new(current_page, per_page, total_items, total_pages, page_len); + + (page_items, pagination) +} + +#[cfg(test)] +mod tests { + use super::paginate_items; + + #[test] + fn paginates_basic_case() { + let items: Vec<_> = (1..=25).collect(); + let (page, meta) = paginate_items(items, Some(2), 10); + + assert_eq!(page, vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]); + assert_eq!(meta.current_page, 2); + assert_eq!(meta.total_pages, 3); + assert!(meta.has_previous); + assert!(meta.has_next); + assert_eq!(meta.previous_page, Some(1)); + assert_eq!(meta.next_page, Some(3)); + assert_eq!(meta.start_index, 11); + assert_eq!(meta.end_index, 20); + } + + #[test] + fn handles_empty_items() { + let items: Vec = vec![]; + let (page, meta) = paginate_items(items, Some(3), 10); + + assert!(page.is_empty()); + assert_eq!(meta.current_page, 1); + assert_eq!(meta.total_pages, 0); + assert!(!meta.has_previous); + assert!(!meta.has_next); + assert_eq!(meta.start_index, 0); + assert_eq!(meta.end_index, 0); + } + + #[test] + fn clamps_page_to_bounds() { + let items: Vec<_> = (1..=5).collect(); + let (page, meta) = paginate_items(items, Some(10), 2); + + assert_eq!(page, vec![5]); + assert_eq!(meta.current_page, 3); + assert_eq!(meta.total_pages, 3); + assert_eq!(meta.has_next, false); + assert_eq!(meta.has_previous, true); + assert_eq!(meta.start_index, 5); + assert_eq!(meta.end_index, 5); + } +} diff --git a/html-router/templates/content/base.html b/html-router/templates/content/base.html index e42bb72..bdbfde7 100644 --- a/html-router/templates/content/base.html +++ b/html-router/templates/content/base.html @@ -9,6 +9,7 @@

Content

+