diff --git a/common/src/storage/types/file_info.rs b/common/src/storage/types/file_info.rs index f556597..477e34d 100644 --- a/common/src/storage/types/file_info.rs +++ b/common/src/storage/types/file_info.rs @@ -11,7 +11,7 @@ use tokio::fs::remove_dir_all; use tracing::info; use uuid::Uuid; -use crate::{storage::db::SurrealDbClient, stored_object}; +use crate::{error::AppError, storage::db::SurrealDbClient, stored_object}; #[derive(Error, Debug)] pub enum FileError { @@ -221,15 +221,15 @@ impl FileInfo { /// /// # Returns /// `Result<(), FileError>` - pub async fn delete_by_id(id: &str, db_client: &SurrealDbClient) -> Result<(), FileError> { + pub async fn delete_by_id(id: &str, db_client: &SurrealDbClient) -> Result<(), AppError> { // Get the FileInfo from the database let file_info = match db_client.get_item::(id).await? { Some(info) => info, None => { - return Err(FileError::FileNotFound(format!( + return Err(AppError::from(FileError::FileNotFound(format!( "File with id {} was not found", id - ))) + )))) } }; @@ -242,15 +242,15 @@ impl FileInfo { remove_dir_all(parent_dir).await?; info!("Removed directory {:?} and its contents", parent_dir); } else { - return Err(FileError::FileNotFound( + return Err(AppError::from(FileError::FileNotFound( "File has no parent directory".to_string(), - )); + ))); } } else { - return Err(FileError::FileNotFound(format!( + return Err(AppError::from(FileError::FileNotFound(format!( "File at path {:?} was not found", file_path - ))); + )))); } // Delete the FileInfo from the database diff --git a/common/src/storage/types/ingestion_payload.rs b/common/src/storage/types/ingestion_payload.rs index e3b5851..336adae 100644 --- a/common/src/storage/types/ingestion_payload.rs +++ b/common/src/storage/types/ingestion_payload.rs @@ -1,5 +1,4 @@ use crate::{error::AppError, storage::types::file_info::FileInfo}; -use chrono::Utc; use serde::{Deserialize, Serialize}; use tracing::info; use url::Url; diff --git a/common/src/storage/types/user.rs b/common/src/storage/types/user.rs index a35399c..c6f3d6f 100644 --- a/common/src/storage/types/user.rs +++ b/common/src/storage/types/user.rs @@ -227,6 +227,50 @@ impl User { Ok(entities) } + pub async fn get_knowledge_entities_by_type( + user_id: &str, + entity_type: &str, + db: &SurrealDbClient, + ) -> Result, AppError> { + let entities: Vec = db + .client + .query("SELECT * FROM type::table($table) WHERE user_id = $user_id AND entity_type = $entity_type") + .bind(("table", KnowledgeEntity::table_name())) + .bind(("user_id", user_id.to_owned())) + .bind(("entity_type", entity_type.to_owned())) + .await? + .take(0)?; + + Ok(entities) + } + + pub async fn get_entity_types( + user_id: &str, + db: &SurrealDbClient, + ) -> Result, AppError> { + #[derive(Deserialize)] + struct EntityTypeResponse { + entity_type: String, + } + + // Query to select distinct entity types for the user + let response: Vec = db + .client + .query("SELECT entity_type FROM type::table($table_name) WHERE user_id = $user_id GROUP BY entity_type") + .bind(("user_id", user_id.to_owned())) + .bind(("table_name", KnowledgeEntity::table_name())) + .await? + .take(0)?; + + // Extract the entity types from the response + let entity_types: Vec = response + .into_iter() + .map(|item| format!("{:?}", item.entity_type)) + .collect(); + + Ok(entity_types) + } + pub async fn get_knowledge_relationships( user_id: &str, db: &SurrealDbClient, @@ -272,6 +316,23 @@ impl User { Ok(items) } + pub async fn get_text_contents_by_category( + user_id: &str, + category: &str, + db: &SurrealDbClient, + ) -> Result, AppError> { + let items: Vec = db + .client + .query("SELECT * FROM type::table($table_name) WHERE user_id = $user_id AND category = $category ORDER BY created_at DESC") + .bind(("user_id", user_id.to_owned())) + .bind(("category", category.to_owned())) + .bind(("table_name", TextContent::table_name())) + .await? + .take(0)?; + + Ok(items) + } + pub async fn get_latest_knowledge_entities( user_id: &str, db: &SurrealDbClient, @@ -413,6 +474,34 @@ impl User { Ok(()) } + + pub async fn get_knowledge_entities_by_content_category( + user_id: &str, + category: &str, + db: &SurrealDbClient, + ) -> Result, AppError> { + // First, find all text content items with the specified category + let text_contents = Self::get_text_contents_by_category(user_id, category, db).await?; + + if text_contents.is_empty() { + return Ok(Vec::new()); + } + + // Extract source_ids + let source_ids: Vec = text_contents.iter().map(|tc| tc.id.clone()).collect(); + + // Find all knowledge entities with matching source_ids + let entities: Vec = db + .client + .query("SELECT * FROM type::table($table) WHERE user_id = $user_id AND source_id IN $source_ids") + .bind(("table", KnowledgeEntity::table_name())) + .bind(("user_id", user_id.to_owned())) + .bind(("source_ids", source_ids)) + .await? + .take(0)?; + + Ok(entities) + } } #[cfg(test)] diff --git a/html-router/src/routes/content/handlers.rs b/html-router/src/routes/content/handlers.rs index ecc3595..58713d8 100644 --- a/html-router/src/routes/content/handlers.rs +++ b/html-router/src/routes/content/handlers.rs @@ -1,11 +1,12 @@ use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, response::IntoResponse, Form, }; +use axum_htmx::{HxBoosted, HxRequest}; use serde::{Deserialize, Serialize}; -use common::storage::types::{text_content::TextContent, user::User}; +use common::storage::types::{file_info::FileInfo, text_content::TextContent, user::User}; use crate::{ html_state::HtmlState, @@ -19,21 +20,52 @@ use crate::{ pub struct ContentPageData { user: User, text_contents: Vec, + categories: Vec, + selected_category: Option, +} + +#[derive(Deserialize)] +pub struct FilterParams { + category: Option, } pub async fn show_content_page( State(state): State, RequireUser(user): RequireUser, + Query(params): Query, + HxRequest(is_htmx): HxRequest, + HxBoosted(is_boosted): HxBoosted, ) -> Result { - let text_contents = User::get_text_contents(&user.id, &state.db).await?; + // Normalize empty strings to None + let has_category_param = params.category.is_some(); + let category_filter = params.category.as_deref().unwrap_or("").trim(); - Ok(TemplateResponse::new_template( - "content/base.html", - ContentPageData { - user, - text_contents, - }, - )) + // load categories and filtered/all contents + let categories = User::get_user_categories(&user.id, &state.db).await?; + let text_contents = if !category_filter.is_empty() { + User::get_text_contents_by_category(&user.id, category_filter, &state.db).await? + } else { + User::get_text_contents(&user.id, &state.db).await? + }; + + let data = ContentPageData { + user, + text_contents, + categories, + selected_category: params.category.clone(), + }; + + if is_htmx && !is_boosted && has_category_param { + // If HTMX partial request with filter applied, return partial content list update + return Ok(TemplateResponse::new_partial( + "content/base.html", + "main", + data, + )); + } + + // Otherwise full page response including layout + Ok(TemplateResponse::new_template("content/base.html", data)) } pub async fn show_text_content_edit_form( @@ -79,12 +111,47 @@ pub async fn patch_text_content( .await?; let text_contents = User::get_text_contents(&user.id, &state.db).await?; + let categories = User::get_user_categories(&user.id, &state.db).await?; + + Ok(TemplateResponse::new_partial( + "content/base.html", + "main", + ContentPageData { + user, + text_contents, + categories, + selected_category: None, + }, + )) +} + +pub async fn delete_text_content( + State(state): State, + RequireUser(user): RequireUser, + Path(id): Path, +) -> Result { + // Get and validate the text content + let text_content = User::get_and_validate_text_content(&id, &user.id, &state.db).await?; + + // If it has file info, delete that too + if let Some(file_info) = &text_content.file_info { + FileInfo::delete_by_id(&file_info.id, &state.db).await?; + } + + // Delete the text content + state.db.delete_item::(&id).await?; + + // Get updated content, categories and return the refreshed list + let text_contents = User::get_text_contents(&user.id, &state.db).await?; + let categories = User::get_user_categories(&user.id, &state.db).await?; Ok(TemplateResponse::new_template( "content/content_list.html", ContentPageData { user, text_contents, + categories, + selected_category: None, }, )) } diff --git a/html-router/src/routes/content/mod.rs b/html-router/src/routes/content/mod.rs index 962dc7f..281346b 100644 --- a/html-router/src/routes/content/mod.rs +++ b/html-router/src/routes/content/mod.rs @@ -1,7 +1,9 @@ mod handlers; use axum::{extract::FromRef, routing::get, Router}; -use handlers::{patch_text_content, show_content_page, show_text_content_edit_form}; +use handlers::{ + delete_text_content, patch_text_content, show_content_page, show_text_content_edit_form, +}; use crate::html_state::HtmlState; @@ -14,6 +16,8 @@ where .route("/content", get(show_content_page)) .route( "/content/:id", - get(show_text_content_edit_form).patch(patch_text_content), + get(show_text_content_edit_form) + .patch(patch_text_content) + .delete(delete_text_content), ) } diff --git a/html-router/src/routes/knowledge/handlers.rs b/html-router/src/routes/knowledge/handlers.rs index 49c82ad..2b2144a 100644 --- a/html-router/src/routes/knowledge/handlers.rs +++ b/html-router/src/routes/knowledge/handlers.rs @@ -1,12 +1,15 @@ +use std::collections::{HashMap, VecDeque}; + use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, response::IntoResponse, Form, }; +use axum_htmx::{HxBoosted, HxRequest}; use plotly::{ common::{Line, Marker, Mode}, - layout::{Axis, Camera, LayoutScene, ProjectionType}, - Layout, Plot, Scatter3D, + layout::{Axis, LayoutScene}, + Layout, Plot, Scatter, Scatter3D, }; use serde::{Deserialize, Serialize}; @@ -24,45 +27,178 @@ use crate::{ }, }; +#[derive(Deserialize, Default)] +pub struct FilterParams { + entity_type: Option, + content_category: Option, +} + +#[derive(Serialize)] +pub struct KnowledgeBaseData { + entities: Vec, + relationships: Vec, + user: User, + plot_html: String, + entity_types: Vec, + content_categories: Vec, + selected_entity_type: Option, + selected_content_category: Option, +} + pub async fn show_knowledge_page( State(state): State, RequireUser(user): RequireUser, + Query(mut params): Query, + HxRequest(is_htmx): HxRequest, + HxBoosted(is_boosted): HxBoosted, ) -> Result { - #[derive(Serialize)] - pub struct KnowledgeBaseData { - entities: Vec, - relationships: Vec, - user: User, - plot_html: String, - } + // Normalize filters + params.entity_type = params.entity_type.take().filter(|s| !s.trim().is_empty()); + params.content_category = params + .content_category + .take() + .filter(|s| !s.trim().is_empty()); - let entities = User::get_knowledge_entities(&user.id, &state.db).await?; + // Load relevant data + let entity_types = User::get_entity_types(&user.id, &state.db).await?; + let content_categories = User::get_user_categories(&user.id, &state.db).await?; + + // Load entities based on filters + let entities = match ¶ms.content_category { + Some(cat) => { + User::get_knowledge_entities_by_content_category(&user.id, cat, &state.db).await? + } + None => match ¶ms.entity_type { + Some(etype) => User::get_knowledge_entities_by_type(&user.id, etype, &state.db).await?, + None => User::get_knowledge_entities(&user.id, &state.db).await?, + }, + }; let relationships = User::get_knowledge_relationships(&user.id, &state.db).await?; + let plot_html = get_plot_html(&entities, &relationships)?; - let mut plot = Plot::new(); + let kb_data = KnowledgeBaseData { + entities, + relationships, + user, + plot_html, + entity_types, + content_categories, + selected_entity_type: params.entity_type.clone(), + selected_content_category: params.content_category.clone(), + }; - // Fibonacci sphere distribution - let node_count = entities.len(); - let golden_ratio = (1.0 + 5.0_f64.sqrt()) / 2.0; - let node_positions: Vec<(f64, f64, f64)> = (0..node_count) - .map(|i| { - let i = i as f64; - let theta = 2.0 * std::f64::consts::PI * i / golden_ratio; - let phi = (1.0 - 2.0 * (i + 0.5) / node_count as f64).acos(); - let x = phi.sin() * theta.cos(); - let y = phi.sin() * theta.sin(); - let z = phi.cos(); - (x, y, z) - }) + // Determine response type: + // If it is an HTMX request but NOT a boosted navigation, send partial update (main block only) + // Otherwise send full page including navbar/base for direct and boosted reloads + if is_htmx && !is_boosted { + // Partial update (just main block) + Ok(TemplateResponse::new_partial( + "knowledge/base.html", + "main", + &kb_data, + )) + } else { + // Full page (includes navbar etc.) + Ok(TemplateResponse::new_template( + "knowledge/base.html", + kb_data, + )) + } +} + +fn get_plot_html( + entities: &[KnowledgeEntity], + relationships: &[KnowledgeRelationship], +) -> Result { + if entities.is_empty() { + return Ok(String::new()); + } + + let id_to_idx: HashMap<_, _> = entities + .iter() + .enumerate() + .map(|(i, e)| (e.id.clone(), i)) .collect(); - let node_x: Vec = node_positions.iter().map(|(x, _, _)| *x).collect(); - let node_y: Vec = node_positions.iter().map(|(_, y, _)| *y).collect(); - let node_z: Vec = node_positions.iter().map(|(_, _, z)| *z).collect(); + // Build adjacency list + let mut graph: Vec> = vec![Vec::new(); entities.len()]; + for rel in relationships { + if let (Some(&from_idx), Some(&to_idx)) = (id_to_idx.get(&rel.out), id_to_idx.get(&rel.in_)) + { + graph[from_idx].push(to_idx); + graph[to_idx].push(from_idx); + } + } + + // Find clusters (connected components) + let mut visited = vec![false; entities.len()]; + let mut clusters: Vec> = Vec::new(); + + for i in 0..entities.len() { + if !visited[i] { + let mut queue = VecDeque::new(); + let mut cluster = Vec::new(); + queue.push_back(i); + visited[i] = true; + while let Some(node) = queue.pop_front() { + cluster.push(node); + for &nbr in &graph[node] { + if !visited[nbr] { + visited[nbr] = true; + queue.push_back(nbr); + } + } + } + clusters.push(cluster); + } + } + + // Layout params + let cluster_spacing = 20.0; // Distance between clusters + let node_spacing = 3.0; // Distance between nodes within cluster + + // Arrange clusters on a Fibonacci sphere (uniform 3D positioning on unit sphere) + let cluster_count = clusters.len(); + let golden_angle = std::f64::consts::PI * (3.0 - (5.0f64).sqrt()); + + // Will hold final positions of nodes: (x,y,z) + let mut nodes_pos = vec![(0.0f64, 0.0f64, 0.0f64); entities.len()]; + + for (i, cluster) in clusters.iter().enumerate() { + // Position cluster center on unit sphere scaled by cluster_spacing + let theta = golden_angle * i as f64; + let z = 1.0 - (2.0 * i as f64 + 1.0) / cluster_count as f64; + let radius = (1.0 - z * z).sqrt(); + + let cluster_center = ( + radius * theta.cos() * cluster_spacing, + radius * theta.sin() * cluster_spacing, + z * cluster_spacing, + ); + + // Layout nodes within cluster as small 3D grid (cube) + // Calculate cube root to determine grid side length + let cluster_size = cluster.len(); + let side_len = (cluster_size as f64).cbrt().ceil() as usize; + + for (pos_in_cluster, &node_idx) in cluster.iter().enumerate() { + let x_in_cluster = (pos_in_cluster % side_len) as f64; + let y_in_cluster = ((pos_in_cluster / side_len) % side_len) as f64; + let z_in_cluster = (pos_in_cluster / (side_len * side_len)) as f64; + + nodes_pos[node_idx] = ( + cluster_center.0 + x_in_cluster * node_spacing, + cluster_center.1 + y_in_cluster * node_spacing, + cluster_center.2 + z_in_cluster * node_spacing, + ); + } + } + + let (node_x, node_y, node_z): (Vec<_>, Vec<_>, Vec<_>) = nodes_pos.iter().cloned().unzip3(); // Nodes trace - let nodes = Scatter3D::new(node_x.clone(), node_y.clone(), node_z.clone()) + let nodes_trace = Scatter3D::new(node_x, node_y, node_z) .mode(Mode::Markers) .marker(Marker::new().size(8).color("#1f77b4")) .text_array( @@ -71,31 +207,32 @@ pub async fn show_knowledge_page( .map(|e| e.description.clone()) .collect::>(), ) - .hover_template("Entity: %{text}
"); + .hover_template("Entity: %{text}"); // Edges traces - for rel in &relationships { - let from_idx = entities.iter().position(|e| e.id == rel.out).unwrap_or(0); - let to_idx = entities.iter().position(|e| e.id == rel.in_).unwrap_or(0); + let mut plot = Plot::new(); + for rel in relationships { + if let (Some(&from_idx), Some(&to_idx)) = (id_to_idx.get(&rel.out), id_to_idx.get(&rel.in_)) + { + let edge_x = vec![nodes_pos[from_idx].0, nodes_pos[to_idx].0]; + let edge_y = vec![nodes_pos[from_idx].1, nodes_pos[to_idx].1]; + let edge_z = vec![nodes_pos[from_idx].2, nodes_pos[to_idx].2]; - let edge_x = vec![node_x[from_idx], node_x[to_idx]]; - let edge_y = vec![node_y[from_idx], node_y[to_idx]]; - let edge_z = vec![node_z[from_idx], node_z[to_idx]]; - - let edge_trace = Scatter3D::new(edge_x, edge_y, edge_z) - .mode(Mode::Lines) - .line(Line::new().color("#888").width(2.0)) - .hover_template(format!( - "Relationship: {}
", - rel.metadata.relationship_type - )) - .show_legend(false); - - plot.add_trace(edge_trace); + let edge_trace = Scatter3D::new(edge_x, edge_y, edge_z) + .mode(Mode::Lines) + .line(Line::new().color("#888").width(2.0)) + .hover_template(format!( + "Relationship: {}", + rel.metadata.relationship_type + )) + .show_legend(false); + plot.add_trace(edge_trace); + } } - plot.add_trace(nodes); - // Layout + plot.add_trace(nodes_trace); + + // Layout scene configuration let layout = Layout::new() .scene( LayoutScene::new() @@ -103,29 +240,37 @@ pub async fn show_knowledge_page( .y_axis(Axis::new().visible(false)) .z_axis(Axis::new().visible(false)) .camera( - Camera::new() - .projection(ProjectionType::Perspective.into()) - .eye((1.5, 1.5, 1.5).into()), + plotly::layout::Camera::new() + .projection(plotly::layout::ProjectionType::Perspective.into()) + .eye((2.0, 2.0, 2.0).into()), ), ) .show_legend(false) - .paper_background_color("rbga(250,100,0,0)") - .plot_background_color("rbga(0,0,0,0)"); + .paper_background_color("rgba(255,255,255,0)") + .plot_background_color("rgba(255,255,255,0)"); plot.set_layout(layout); - // Convert to HTML - let html = plot.to_html(); + Ok(plot.to_html()) +} - Ok(TemplateResponse::new_template( - "knowledge/base.html", - KnowledgeBaseData { - entities, - relationships, - user, - plot_html: html, - }, - )) +// Small utility to unzip tuple3 vectors from iterators (add this helper) +trait Unzip3 { + fn unzip3(self) -> (Vec, Vec, Vec); +} +impl Unzip3 for I +where + I: Iterator, +{ + fn unzip3(self) -> (Vec, Vec, Vec) { + let (mut va, mut vb, mut vc) = (Vec::new(), Vec::new(), Vec::new()); + for (a, b, c) in self { + va.push(a); + vb.push(b); + vc.push(c); + } + (va, vb, vc) + } } pub async fn show_edit_knowledge_entity_form( @@ -171,6 +316,10 @@ pub struct PatchKnowledgeEntityParams { pub struct EntityListData { entities: Vec, user: User, + entity_types: Vec, + content_categories: Vec, + selected_entity_type: Option, + selected_content_category: Option, } pub async fn patch_knowledge_entity( @@ -197,10 +346,23 @@ pub async fn patch_knowledge_entity( // Get updated list of entities let entities = User::get_knowledge_entities(&user.id, &state.db).await?; + // Get entity types + let entity_types = User::get_entity_types(&user.id, &state.db).await?; + + // Get content categories + let content_categories = User::get_user_categories(&user.id, &state.db).await?; + // Render updated list Ok(TemplateResponse::new_template( "knowledge/entity_list.html", - EntityListData { entities, user }, + EntityListData { + entities, + user, + entity_types, + content_categories, + selected_entity_type: None, + selected_content_category: None, + }, )) } @@ -218,9 +380,22 @@ pub async fn delete_knowledge_entity( // Get updated list of entities let entities = User::get_knowledge_entities(&user.id, &state.db).await?; + // Get entity types + let entity_types = User::get_entity_types(&user.id, &state.db).await?; + + // Get content categories + let content_categories = User::get_user_categories(&user.id, &state.db).await?; + Ok(TemplateResponse::new_template( "knowledge/entity_list.html", - EntityListData { entities, user }, + EntityListData { + entities, + user, + entity_types, + content_categories, + selected_entity_type: None, + selected_content_category: None, + }, )) } diff --git a/html-router/templates/content/base.html b/html-router/templates/content/base.html index 5be28c1..1cd9e4d 100644 --- a/html-router/templates/content/base.html +++ b/html-router/templates/content/base.html @@ -1,12 +1,27 @@ {% extends 'body_base.html' %} {% block main %} -
+
-

Text Contents

- {% include "content/content_list.html" %} - - +
+

Text Contents

+
+
+ +
+ +
+
+
+ {% include "content/content_list.html" %} +
{% endblock %} \ No newline at end of file diff --git a/html-router/templates/content/content_list.html b/html-router/templates/content/content_list.html index e88f808..e24bc0b 100644 --- a/html-router/templates/content/content_list.html +++ b/html-router/templates/content/content_list.html @@ -2,16 +2,7 @@ {% for text_content in text_contents %}
-
-
- {% if text_content.url %} - {% include "icons/globe_icon.html" %} - {% elif text_content.file_info %} - {% include "icons/document_icon.html" %} - {% else %} - {% include "icons/chat_icon.html" %} - {% endif %} -
+

{% if text_content.url %} {{text_content.url}} @@ -21,11 +12,21 @@ {{text_content.text}} {% endif %}

+
+ {% if text_content.url %} + {% include "icons/globe_icon.html" %} + {% elif text_content.file_info %} + {% include "icons/document_icon.html" %} + {% else %} + {% include "icons/chat_icon.html" %} + {% endif %} +
-
+

{{ text_content.created_at | datetimeformat(format="short", tz=user.timezone) }}

+
{{ text_content.category }}
- {{item.created_at|datetimeformat(format="short", tz=user.timezone)}}
+ {{item.created_at|datetimeformat(format="short", tz=user.timezone)}} + {{item.category}} +

{{item.instructions}} diff --git a/html-router/templates/knowledge/base.html b/html-router/templates/knowledge/base.html index 4d0d4ef..ca25719 100644 --- a/html-router/templates/knowledge/base.html +++ b/html-router/templates/knowledge/base.html @@ -1,19 +1,38 @@ {% extends 'body_base.html' %} {% block main %} -

+
-

Entities

- {% include "knowledge/entity_list.html" %} - - - -

Relationships

- {% include "knowledge/relationship_table.html" %} - -
- {{plot_html|safe}} +
+

Entities

+
+
+ +
+
+ +
+ +
+ {% include "knowledge/entity_list.html" %} +

Relationships

+ {% include "knowledge/relationship_table.html" %} +
+ {{ plot_html | safe }} +
{% endblock %} \ No newline at end of file diff --git a/html-router/templates/knowledge/entity_list.html b/html-router/templates/knowledge/entity_list.html index 3b211cb..de27c7e 100644 --- a/html-router/templates/knowledge/entity_list.html +++ b/html-router/templates/knowledge/entity_list.html @@ -6,10 +6,10 @@ {{entity.entity_type}}
-

{{entity.updated_at | datetimeformat(format="short", tz=user.timezeone)}}

+

{{entity.updated_at | datetimeformat(format="short", tz=user.timezone)}}