From eb928cdb0eeafb6bd713120993c8ad7a722c23fc Mon Sep 17 00:00:00 2001 From: Per Stark Date: Sun, 15 Feb 2026 08:52:56 +0100 Subject: [PATCH] test: minio to devenv, improved testing s3 and relationships --- common/src/storage/store.rs | 46 +++- .../storage/types/knowledge_relationship.rs | 196 +++++++++++------- devenv.nix | 14 ++ html-router/assets/style.css | 90 ++++++++ 4 files changed, 264 insertions(+), 82 deletions(-) diff --git a/common/src/storage/store.rs b/common/src/storage/store.rs index e5c5027..bb78034 100644 --- a/common/src/storage/store.rs +++ b/common/src/storage/store.rs @@ -281,6 +281,33 @@ pub mod testing { use crate::utils::config::{AppConfig, PdfIngestMode}; use uuid; + const DEFAULT_TEST_S3_BUCKET: &str = "minne-tests"; + const DEFAULT_TEST_S3_ENDPOINT: &str = "http://127.0.0.1:19000"; + + fn configured_test_s3_bucket() -> String { + std::env::var("MINNE_TEST_S3_BUCKET") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + std::env::var("S3_BUCKET") + .ok() + .filter(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| DEFAULT_TEST_S3_BUCKET.to_string()) + } + + fn configured_test_s3_endpoint() -> String { + std::env::var("MINNE_TEST_S3_ENDPOINT") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + std::env::var("S3_ENDPOINT") + .ok() + .filter(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| DEFAULT_TEST_S3_ENDPOINT.to_string()) + } + /// Create a test configuration with memory storage. /// /// This provides a ready-to-use configuration for testing scenarios @@ -326,7 +353,8 @@ pub mod testing { /// Create a test configuration with S3 storage (MinIO). /// - /// This requires a running MinIO instance on localhost:9000. + /// Uses `MINNE_TEST_S3_ENDPOINT` / `S3_ENDPOINT` and + /// `MINNE_TEST_S3_BUCKET` / `S3_BUCKET` when provided. pub fn test_config_s3() -> AppConfig { AppConfig { openai_api_key: "test".into(), @@ -339,8 +367,8 @@ pub mod testing { http_port: 0, openai_base_url: "..".into(), storage: StorageKind::S3, - s3_bucket: Some("minne-tests".into()), - s3_endpoint: Some("http://localhost:9000".into()), + s3_bucket: Some(configured_test_s3_bucket()), + s3_endpoint: Some(configured_test_s3_endpoint()), s3_region: Some("us-east-1".into()), pdf_ingest_mode: PdfIngestMode::LlmFirst, ..Default::default() @@ -391,8 +419,7 @@ pub mod testing { /// Create a new TestStorageManager with S3 backend (MinIO). /// - /// This requires a running MinIO instance on localhost:9000 with - /// default credentials (minioadmin/minioadmin) and a 'minne-tests' bucket. + /// This requires a reachable MinIO endpoint and an existing test bucket. pub async fn new_s3() -> object_store::Result { // Ensure credentials are set for MinIO // We set these env vars for the process, which AmazonS3Builder will pick up @@ -403,6 +430,11 @@ pub mod testing { let cfg = test_config_s3(); let storage = StorageManager::new(&cfg).await?; + // Probe the bucket so tests can cleanly skip when the endpoint is unreachable + // or the test bucket is not provisioned. + let probe_prefix = format!("__minne_s3_probe__/{}", uuid::Uuid::new_v4()); + storage.list(Some(&probe_prefix)).await?; + Ok(Self { storage, _temp_dir: None, @@ -923,8 +955,8 @@ mod tests { assert_eq!(*test_storage.storage().backend_kind(), StorageKind::Memory); } - // S3 Tests - Require MinIO on localhost:9000 with bucket 'minne-tests' - // These tests will fail if MinIO is not running or bucket doesn't exist. + // S3 Tests - Require a reachable MinIO endpoint and test bucket. + // `TestStorageManager::new_s3()` probes connectivity and these tests auto-skip when unavailable. #[tokio::test] async fn test_storage_manager_s3_basic_operations() { diff --git a/common/src/storage/types/knowledge_relationship.rs b/common/src/storage/types/knowledge_relationship.rs index 664588b..82134cd 100644 --- a/common/src/storage/types/knowledge_relationship.rs +++ b/common/src/storage/types/knowledge_relationship.rs @@ -72,7 +72,7 @@ impl KnowledgeRelationship { ) -> Result<(), AppError> { db_client .client - .query("DELETE knowledge_entity -> relates_to WHERE metadata.source_id = $source_id") + .query("DELETE FROM relates_to WHERE metadata.source_id = $source_id") .bind(("source_id", source_id.to_owned())) .await? .check()?; @@ -127,6 +127,34 @@ mod tests { use super::*; use crate::storage::types::knowledge_entity::{KnowledgeEntity, KnowledgeEntityType}; + async fn setup_test_db() -> SurrealDbClient { + let namespace = "test_ns"; + let database = &Uuid::new_v4().to_string(); + let db = SurrealDbClient::memory(namespace, database) + .await + .expect("Failed to start in-memory surrealdb"); + + db.apply_migrations() + .await + .expect("Failed to apply migrations"); + + db + } + + async fn get_relationship_by_id( + relationship_id: &str, + db_client: &SurrealDbClient, + ) -> Option { + let mut result = db_client + .client + .query("SELECT * FROM type::thing('relates_to', $id)") + .bind(("id", relationship_id.to_owned())) + .await + .expect("relationship query by id failed"); + + result.take(0).expect("failed to take relationship by id") + } + // Helper function to create a test knowledge entity for the relationship tests async fn create_test_entity(name: &str, db_client: &SurrealDbClient) -> String { let source_id = "source123".to_string(); @@ -178,15 +206,7 @@ mod tests { #[tokio::test] async fn test_store_and_verify_by_source_id() { // Setup in-memory database for testing - let namespace = "test_ns"; - let database = &Uuid::new_v4().to_string(); - let db = SurrealDbClient::memory(namespace, database) - .await - .expect("Failed to start in-memory surrealdb"); - - db.apply_migrations() - .await - .expect("Failed to apply migrations"); + let db = setup_test_db().await; // Create two entities to relate let entity1_id = create_test_entity("Entity 1", &db).await; @@ -211,33 +231,33 @@ mod tests { .await .expect("Failed to store relationship"); + let persisted = get_relationship_by_id(&relationship.id, &db) + .await + .expect("Relationship should be retrievable by id"); + assert_eq!(persisted.in_, entity1_id); + assert_eq!(persisted.out, entity2_id); + assert_eq!(persisted.metadata.user_id, user_id); + assert_eq!(persisted.metadata.source_id, source_id); + // Query to verify the relationship exists by checking for relationships with our source_id // This approach is more reliable than trying to look up by ID - let check_query = format!( - "SELECT * FROM relates_to WHERE metadata.source_id = '{}'", - source_id - ); - let mut check_result = db.query(check_query).await.expect("Check query failed"); + let mut check_result = db + .query("SELECT * FROM relates_to WHERE metadata.source_id = $source_id") + .bind(("source_id", source_id.clone())) + .await + .expect("Check query failed"); let check_results: Vec = check_result.take(0).unwrap_or_default(); - // Just verify that a relationship was created - assert!( - !check_results.is_empty(), - "Relationship should exist in the database" + assert_eq!( + check_results.len(), + 1, + "Expected one relationship for source_id" ); } #[tokio::test] async fn test_store_relationship_resists_query_injection() { - let namespace = "test_ns"; - let database = &Uuid::new_v4().to_string(); - let db = SurrealDbClient::memory(namespace, database) - .await - .expect("Failed to start in-memory surrealdb"); - - db.apply_migrations() - .await - .expect("Failed to apply migrations"); + let db = setup_test_db().await; let entity1_id = create_test_entity("Entity 1", &db).await; let entity2_id = create_test_entity("Entity 2", &db).await; @@ -273,11 +293,7 @@ mod tests { #[tokio::test] async fn test_store_and_delete_relationship() { // Setup in-memory database for testing - let namespace = "test_ns"; - let database = &Uuid::new_v4().to_string(); - let db = SurrealDbClient::memory(namespace, database) - .await - .expect("Failed to start in-memory surrealdb"); + let db = setup_test_db().await; // Create two entities to relate let entity1_id = create_test_entity("Entity 1", &db).await; @@ -338,11 +354,7 @@ mod tests { #[tokio::test] async fn test_delete_relationship_by_id_unauthorized() { - let namespace = "test_ns"; - let database = &Uuid::new_v4().to_string(); - let db = SurrealDbClient::memory(namespace, database) - .await - .expect("Failed to start in-memory surrealdb"); + let db = setup_test_db().await; let entity1_id = create_test_entity("Entity 1", &db).await; let entity2_id = create_test_entity("Entity 2", &db).await; @@ -406,11 +418,7 @@ mod tests { #[tokio::test] async fn test_store_relationship_exists() { // Setup in-memory database for testing - let namespace = "test_ns"; - let database = &Uuid::new_v4().to_string(); - let db = SurrealDbClient::memory(namespace, database) - .await - .expect("Failed to start in-memory surrealdb"); + let db = setup_test_db().await; // Create entities to relate let entity1_id = create_test_entity("Entity 1", &db).await; @@ -462,49 +470,87 @@ mod tests { .await .expect("Failed to store different relationship"); + // Sanity-check setup: exactly two relationships use source_id and one uses different_source_id. + let mut before_delete = db + .query("SELECT * FROM relates_to WHERE metadata.source_id = $source_id") + .bind(("source_id", source_id.clone())) + .await + .expect("before delete query failed"); + let before_delete_rows: Vec = + before_delete.take(0).unwrap_or_default(); + assert_eq!(before_delete_rows.len(), 2); + + let mut before_delete_different = db + .query("SELECT * FROM relates_to WHERE metadata.source_id = $source_id") + .bind(("source_id", different_source_id.clone())) + .await + .expect("before delete different query failed"); + let before_delete_different_rows: Vec = + before_delete_different.take(0).unwrap_or_default(); + assert_eq!(before_delete_different_rows.len(), 1); + // Delete relationships by source_id KnowledgeRelationship::delete_relationships_by_source_id(&source_id, &db) .await .expect("Failed to delete relationships by source_id"); - // Query to verify the relationships with source_id were deleted - let query1 = format!("SELECT * FROM relates_to WHERE id = '{}'", relationship1.id); - let query2 = format!("SELECT * FROM relates_to WHERE id = '{}'", relationship2.id); - let different_query = format!( - "SELECT * FROM relates_to WHERE id = '{}'", - different_relationship.id - ); - - let mut result1 = db.query(query1).await.expect("Query 1 failed"); - let results1: Vec = result1.take(0).unwrap_or_default(); - - let mut result2 = db.query(query2).await.expect("Query 2 failed"); - let results2: Vec = result2.take(0).unwrap_or_default(); - - let mut different_result = db - .query(different_query) - .await - .expect("Different query failed"); - let _different_results: Vec = - different_result.take(0).unwrap_or_default(); + // Query to verify the specific relationships with source_id were deleted. + let result1 = get_relationship_by_id(&relationship1.id, &db).await; + let result2 = get_relationship_by_id(&relationship2.id, &db).await; + let different_result = get_relationship_by_id(&different_relationship.id, &db).await; // Verify relationships with the source_id are deleted - assert!(results1.is_empty(), "Relationship 1 should be deleted"); - assert!(results2.is_empty(), "Relationship 2 should be deleted"); + assert!(result1.is_none(), "Relationship 1 should be deleted"); + assert!(result2.is_none(), "Relationship 2 should be deleted"); + let remaining = + different_result.expect("Relationship with different source_id should remain"); + assert_eq!(remaining.metadata.source_id, different_source_id); + } - // For the relationship with different source ID, we need to check differently - // Let's just verify we have a relationship where the source_id matches different_source_id - let check_query = format!( - "SELECT * FROM relates_to WHERE metadata.source_id = '{}'", - different_source_id + #[tokio::test] + async fn test_delete_relationships_by_source_id_resists_query_injection() { + let db = setup_test_db().await; + + let entity1_id = create_test_entity("Entity 1", &db).await; + let entity2_id = create_test_entity("Entity 2", &db).await; + let entity3_id = create_test_entity("Entity 3", &db).await; + + let safe_relationship = KnowledgeRelationship::new( + entity1_id.clone(), + entity2_id.clone(), + "user123".to_string(), + "safe_source".to_string(), + "references".to_string(), ); - let mut check_result = db.query(check_query).await.expect("Check query failed"); - let check_results: Vec = check_result.take(0).unwrap_or_default(); - // Verify the relationship with a different source_id still exists + let other_relationship = KnowledgeRelationship::new( + entity2_id, + entity3_id, + "user123".to_string(), + "other_source".to_string(), + "contains".to_string(), + ); + + safe_relationship + .store_relationship(&db) + .await + .expect("store safe relationship"); + other_relationship + .store_relationship(&db) + .await + .expect("store other relationship"); + + KnowledgeRelationship::delete_relationships_by_source_id("safe_source' OR 1=1 --", &db) + .await + .expect("delete call should succeed"); + + let remaining_safe = get_relationship_by_id(&safe_relationship.id, &db).await; + let remaining_other = get_relationship_by_id(&other_relationship.id, &db).await; + + assert!(remaining_safe.is_some(), "Safe relationship should remain"); assert!( - !check_results.is_empty(), - "Relationship with different source_id should still exist" + remaining_other.is_some(), + "Other relationship should remain" ); } } diff --git a/devenv.nix b/devenv.nix index dca78f4..2f0a570 100644 --- a/devenv.nix +++ b/devenv.nix @@ -30,6 +30,20 @@ env = { ORT_DYLIB_PATH = "${pkgs.onnxruntime}/lib/libonnxruntime.so"; + S3_ENDPOINT = "http://127.0.0.1:19000"; + S3_BUCKET = "minne-tests"; + MINNE_TEST_S3_ENDPOINT = "http://127.0.0.1:19000"; + MINNE_TEST_S3_BUCKET = "minne-tests"; + }; + + services.minio = { + enable = true; + listenAddress = "127.0.0.1:19000"; + consoleAddress = "127.0.0.1:19001"; + buckets = ["minne-tests"]; + accessKey = "minioadmin"; + secretKey = "minioadmin"; + region = "us-east-1"; }; processes = { diff --git a/html-router/assets/style.css b/html-router/assets/style.css index 484ae07..3b94d07 100644 --- a/html-router/assets/style.css +++ b/html-router/assets/style.css @@ -285,6 +285,37 @@ } } } + .drawer-open { + > .drawer-side { + overflow-y: auto; + } + > .drawer-toggle { + display: none; + & ~ .drawer-side { + pointer-events: auto; + visibility: visible; + position: sticky; + display: block; + width: auto; + overscroll-behavior: auto; + opacity: 100%; + & > .drawer-overlay { + cursor: default; + background-color: transparent; + } + & > *:not(.drawer-overlay) { + translate: 0%; + [dir="rtl"] & { + translate: 0%; + } + } + } + &:checked ~ .drawer-side { + pointer-events: auto; + visibility: visible; + } + } + } .drawer-toggle { position: fixed; height: calc(0.25rem * 0); @@ -1043,6 +1074,22 @@ grid-row-start: 1; min-width: calc(0.25rem * 0); } + .chat-image { + grid-row: span 2 / span 2; + align-self: flex-end; + } + .chat-footer { + grid-row-start: 3; + display: flex; + gap: calc(0.25rem * 1); + font-size: 0.6875rem; + } + .chat-header { + grid-row-start: 1; + display: flex; + gap: calc(0.25rem * 1); + font-size: 0.6875rem; + } .container { width: 100%; @media (width >= 40rem) { @@ -1749,6 +1796,9 @@ .w-10 { width: calc(var(--spacing) * 10); } + .w-11 { + width: calc(var(--spacing) * 11); + } .w-11\/12 { width: calc(11/12 * 100%); } @@ -1812,6 +1862,9 @@ .flex-none { flex: none; } + .flex-shrink { + flex-shrink: 1; + } .flex-shrink-0 { flex-shrink: 0; } @@ -1824,6 +1877,13 @@ .grow { flex-grow: 1; } + .border-collapse { + border-collapse: collapse; + } + .-translate-y-1 { + --tw-translate-y: calc(var(--spacing) * -1); + translate: var(--tw-translate-x) var(--tw-translate-y); + } .-translate-y-1\/2 { --tw-translate-y: calc(calc(1/2 * 100%) * -1); translate: var(--tw-translate-x) var(--tw-translate-y); @@ -1896,6 +1956,9 @@ .justify-start { justify-content: flex-start; } + .gap-0 { + gap: calc(var(--spacing) * 0); + } .gap-0\.5 { gap: calc(var(--spacing) * 0.5); } @@ -2052,6 +2115,9 @@ .bg-transparent { background-color: transparent; } + .bg-warning { + background-color: var(--color-warning); + } .bg-warning\/10 { background-color: var(--color-warning); @supports (color: color-mix(in lab, red, red)) { @@ -2070,6 +2136,9 @@ .loading-spinner { mask-image: url("data:image/svg+xml,%3Csvg width='24' height='24' stroke='black' viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Cg transform-origin='center'%3E%3Ccircle cx='12' cy='12' r='9.5' fill='none' stroke-width='3' stroke-linecap='round'%3E%3CanimateTransform attributeName='transform' type='rotate' from='0 12 12' to='360 12 12' dur='2s' repeatCount='indefinite'/%3E%3Canimate attributeName='stroke-dasharray' values='0,150;42,150;42,150' keyTimes='0;0.475;1' dur='1.5s' repeatCount='indefinite'/%3E%3Canimate attributeName='stroke-dashoffset' values='0;-16;-59' keyTimes='0;0.475;1' dur='1.5s' repeatCount='indefinite'/%3E%3C/circle%3E%3C/g%3E%3C/svg%3E"); } + .mask-repeat { + mask-repeat: repeat; + } .fill-current { fill: currentcolor; } @@ -2100,6 +2169,9 @@ .p-8 { padding: calc(var(--spacing) * 8); } + .px-1 { + padding-inline: calc(var(--spacing) * 1); + } .px-1\.5 { padding-inline: calc(var(--spacing) * 1.5); } @@ -2254,6 +2326,9 @@ --tw-tracking: var(--tracking-widest); letter-spacing: var(--tracking-widest); } + .text-wrap { + text-wrap: wrap; + } .break-words { overflow-wrap: break-word; } @@ -2320,6 +2395,17 @@ .italic { font-style: italic; } + .underline { + text-decoration-line: underline; + } + .swap-active { + .swap-off { + opacity: 0%; + } + .swap-on { + opacity: 100%; + } + } .opacity-0 { opacity: 0%; } @@ -2410,6 +2496,10 @@ --tw-duration: 300ms; transition-duration: 300ms; } + .ease-in-out { + --tw-ease: var(--ease-in-out); + transition-timing-function: var(--ease-in-out); + } .ease-out { --tw-ease: var(--ease-out); transition-timing-function: var(--ease-out);