release: 0.1.1

fix: docker build dep

.github/workflows/release.yml

@@ -16,6 +16,7 @@
 name: Release
 permissions:
   "contents": "write"
+  "packages": "write"
 
 # This task will run whenever you push a git tag that looks like a version
 # like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc.
@@ -163,6 +164,48 @@ jobs:
             ${{ steps.cargo-dist.outputs.paths }}
             ${{ env.BUILD_MANIFEST_NAME }}
 
+  build_and_push_docker_image:
+    name: Build and Push Docker Image
+    runs-on: ubuntu-latest
+    needs: [plan] 
+    if: ${{ needs.plan.outputs.publishing == 'true' }} 
+    permissions:
+      contents: read      # Permission to checkout the repository
+      packages: write    # Permission to push Docker image to GHCR
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive # Matches your other checkout steps
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }} # User triggering the workflow
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository }} 
+          # This action automatically uses the Git tag as the Docker image tag.
+          # For example, a Git tag 'v1.2.3' will result in Docker tag 'ghcr.io/owner/repo:v1.2.3'.
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: . 
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha # Enable Docker layer caching from GitHub Actions cache
+          cache-to: type=gha,mode=max # Enable Docker layer caching to GitHub Actions cache
+
   # Build and package all the platform-agnostic(ish) things
   build-global-artifacts:
     needs:

Cargo.lock

@@ -3126,7 +3126,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "main"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "anyhow",
  "api-router",

api-router/src/api_state.rs

@@ -5,6 +5,7 @@ use common::{storage::db::SurrealDbClient, utils::config::AppConfig};
 #[derive(Clone)]
 pub struct ApiState {
     pub db: Arc<SurrealDbClient>,
+    pub config: AppConfig,
 }
 
 impl ApiState {
@@ -24,6 +25,7 @@ impl ApiState {
 
         let app_state = ApiState {
             db: surreal_db_client.clone(),
+            config: config.clone(),
         };
 
         Ok(app_state)

api-router/src/routes/ingress.rs

@@ -16,7 +16,7 @@ use crate::{api_state::ApiState, error::ApiError};
 #[derive(Debug, TryFromMultipart)]
 pub struct IngestParams {
     pub content: Option<String>,
-    pub instructions: String,
+    pub context: String,
     pub category: String,
     #[form_data(limit = "10000000")] // Adjust limit as needed
     #[form_data(default)]
@@ -30,17 +30,14 @@ pub async fn ingest_data(
 ) -> Result<impl IntoResponse, ApiError> {
     info!("Received input: {:?}", input);
 
-    let file_infos = try_join_all(
-        input
-            .files
-            .into_iter()
-            .map(|file| FileInfo::new(file, &state.db, &user.id).map_err(AppError::from)),
-    )
+    let file_infos = try_join_all(input.files.into_iter().map(|file| {
+        FileInfo::new(file, &state.db, &user.id, &state.config).map_err(AppError::from)
+    }))
     .await?;
 
     let payloads = IngestionPayload::create_ingestion_payload(
         input.content,
-        input.instructions,
+        input.context,
         input.category,
         file_infos,
         user.id.as_str(),

common/migrations/20250503_215025_initial_setup.surql

@@ -1,15 +1,19 @@
 -- Ensure 'analytics:current' record exists
-CREATE analytics:current CONTENT {
-    page_loads: 0,
-    visitors: 0,
-};
+IF NOT (SELECT * FROM analytics:current) THEN
+    CREATE analytics:current CONTENT {
+        page_loads: 0,
+        visitors: 0
+    };
+END;
 
 -- Ensure 'system_settings:current' record exists
+IF NOT (SELECT * FROM system_settings:current) THEN
 CREATE system_settings:current CONTENT {
     registrations_enabled: true,
     require_email_verification: false,
     query_model: "gpt-4o-mini",
     processing_model: "gpt-4o-mini",
     query_system_prompt: "You are a knowledgeable assistant with access to a specialized knowledge base. You will be provided with relevant knowledge entities from the database as context. Each knowledge entity contains a name, description, and type, representing different concepts, ideas, and information.\nYour task is to:\n1. Carefully analyze the provided knowledge entities in the context\n2. Answer user questions based on this information\n3. Provide clear, concise, and accurate responses\n4. When referencing information, briefly mention which knowledge entity it came from\n5. If the provided context doesn't contain enough information to answer the question confidently, clearly state this\n6. If only partial information is available, explain what you can answer and what information is missing\n7. Avoid making assumptions or providing information not supported by the context\n8. Output the references to the documents. Use the UUIDs and make sure they are correct!\nRemember:\n- Be direct and honest about the limitations of your knowledge\n- Cite the relevant knowledge entities when providing information, but only provide the UUIDs in the reference array\n- If you need to combine information from multiple entities, explain how they connect\n- Don't speculate beyond what's provided in the context\nExample response formats:\n\"Based on [Entity Name], [answer...]\"\n\"I found relevant information in multiple entries: [explanation...]\"\n\"I apologize, but the provided context doesn't contain information about [topic]\"",
-    ingestion_system_prompt: "You are an AI assistant. You will receive a text content, along with user instructions and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users instructions and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.\nThe JSON should have the following structure:\n{\n\"knowledge_entities\": [\n{\n\"key\": \"unique-key-1\",\n\"name\": \"Entity Name\",\n\"description\": \"A detailed description of the entity.\",\n\"entity_type\": \"TypeOfEntity\"\n},\n// More entities...\n],\n\"relationships\": [\n{\n\"type\": \"RelationshipType\",\n\"source\": \"unique-key-1 or UUID from existing database\",\n\"target\": \"unique-key-1 or UUID from existing database\"\n},\n// More relationships...\n]\n}\nGuidelines:\n1. Do NOT generate any IDs or UUIDs. Use a unique `key` for each knowledge entity.\n2. Each KnowledgeEntity should have a unique `key`, a meaningful `name`, and a descriptive `description`.\n3. Define the type of each KnowledgeEntity using the following categories: Idea, Project, Document, Page, TextSnippet.\n4. Establish relationships between entities using types like RelatedTo, RelevantTo, SimilarTo.\n5. Use the `source` key to indicate the originating entity and the `target` key to indicate the related entity\"\n6. You will be presented with a few existing KnowledgeEntities that are similar to the current ones. They will have an existing UUID. When creating relationships to these entities, use their UUID.\n7. Only create relationships between existing KnowledgeEntities.\n8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.\n9. A new relationship MUST include a newly created KnowledgeEntity."
+    ingestion_system_prompt: "You are an AI assistant. You will receive a text content, along with user context and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users context and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.\nThe JSON should have the following structure:\n{\n\"knowledge_entities\": [\n{\n\"key\": \"unique-key-1\",\n\"name\": \"Entity Name\",\n\"description\": \"A detailed description of the entity.\",\n\"entity_type\": \"TypeOfEntity\"\n},\n// More entities...\n],\n\"relationships\": [\n{\n\"type\": \"RelationshipType\",\n\"source\": \"unique-key-1 or UUID from existing database\",\n\"target\": \"unique-key-1 or UUID from existing database\"\n},\n// More relationships...\n]\n}\nGuidelines:\n1. Do NOT generate any IDs or UUIDs. Use a unique `key` for each knowledge entity.\n2. Each KnowledgeEntity should have a unique `key`, a meaningful `name`, and a descriptive `description`.\n3. Define the type of each KnowledgeEntity using the following categories: Idea, Project, Document, Page, TextSnippet.\n4. Establish relationships between entities using types like RelatedTo, RelevantTo, SimilarTo.\n5. Use the `source` key to indicate the originating entity and the `target` key to indicate the related entity\"\n6. You will be presented with a few existing KnowledgeEntities that are similar to the current ones. They will have an existing UUID. When creating relationships to these entities, use their UUID.\n7. Only create relationships between existing KnowledgeEntities.\n8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.\n9. A new relationship MUST include a newly created KnowledgeEntity."
 };
+END;

common/migrations/20250509_152033_rename_instructions_to_context.surql

@@ -0,0 +1,8 @@
+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;
+DEFINE FIELD OVERWRITE instructions ON text_content TYPE option<string>;
+
+UPDATE text_content SET context = instructions WHERE instructions IS NOT NONE;
+
+UPDATE text_content UNSET instructions;
+
+REMOVE FIELD instructions ON TABLE text_content;

common/migrations/definitions/20250509_152033_rename_instructions_to_context.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -198,11 +198,11 @@\n DEFINE FIELD IF NOT EXISTS file_info ON text_content TYPE option<object>;\n # UrlInfo is a struct, store as object\n DEFINE FIELD IF NOT EXISTS url_info ON text_content TYPE option<object>;\n-DEFINE FIELD IF NOT EXISTS instructions ON text_content TYPE string;\n+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;\n DEFINE FIELD IF NOT EXISTS category ON text_content TYPE string;\n DEFINE FIELD IF NOT EXISTS user_id ON text_content TYPE string;\n\n-# Indexes based on query patterns (get_latest_text_contents, get_text_contents_by_category)\n+# Indexes based on query patterns\n DEFINE INDEX IF NOT EXISTS text_content_user_id_idx ON text_content FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS text_content_created_at_idx ON text_content FIELDS created_at;\n DEFINE INDEX IF NOT EXISTS text_content_category_idx ON text_content FIELDS category;\n","events":null}

common/schemas/text_content.surql

@@ -12,11 +12,11 @@ DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;
 DEFINE FIELD IF NOT EXISTS file_info ON text_content TYPE option<object>;
 # UrlInfo is a struct, store as object
 DEFINE FIELD IF NOT EXISTS url_info ON text_content TYPE option<object>;
-DEFINE FIELD IF NOT EXISTS instructions ON text_content TYPE string;
+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;
 DEFINE FIELD IF NOT EXISTS category ON text_content TYPE string;
 DEFINE FIELD IF NOT EXISTS user_id ON text_content TYPE string;
 
-# Indexes based on query patterns (get_latest_text_contents, get_text_contents_by_category)
+# Indexes based on query patterns
 DEFINE INDEX IF NOT EXISTS text_content_user_id_idx ON text_content FIELDS user_id;
 DEFINE INDEX IF NOT EXISTS text_content_created_at_idx ON text_content FIELDS created_at;
 DEFINE INDEX IF NOT EXISTS text_content_category_idx ON text_content FIELDS category;

common/src/storage/types/file_info.rs

@@ -11,7 +11,9 @@ use tokio::fs::remove_dir_all;
 use tracing::info;
 use uuid::Uuid;
 
-use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
+use crate::{
+    error::AppError, storage::db::SurrealDbClient, stored_object, utils::config::AppConfig,
+};
 
 #[derive(Error, Debug)]
 pub enum FileError {
@@ -47,7 +49,9 @@ impl FileInfo {
         field_data: FieldData<NamedTempFile>,
         db_client: &SurrealDbClient,
         user_id: &str,
+        config: &AppConfig,
     ) -> Result<Self, FileError> {
+        info!("Data_dir: {:?}", config);
         let file = field_data.contents;
         let file_name = field_data
             .metadata
@@ -79,7 +83,7 @@ impl FileInfo {
             updated_at: now,
             file_name,
             sha256,
-            path: Self::persist_file(&uuid, file, &sanitized_file_name, user_id)
+            path: Self::persist_file(&uuid, file, &sanitized_file_name, user_id, config)
                 .await?
                 .to_string_lossy()
                 .into(),
@@ -161,13 +165,14 @@ impl FileInfo {
         }
     }
 
-    /// Persists the file to the filesystem under `./data/{user_id}/{uuid}/{file_name}`.
+    /// Persists the file to the filesystem under `{data_dir}/{user_id}/{uuid}/{file_name}`.
     ///
     /// # Arguments
     /// * `uuid` - The UUID of the file.
     /// * `file` - The temporary file to persist.
     /// * `file_name` - The sanitized file name.
     /// * `user-id` - User id
+    /// * `config` - Application configuration containing data directory path
     ///
     /// # Returns
     /// * `Result<PathBuf, FileError>` - The persisted file path or an error.
@@ -176,8 +181,18 @@ impl FileInfo {
         file: NamedTempFile,
         file_name: &str,
         user_id: &str,
+        config: &AppConfig,
     ) -> Result<PathBuf, FileError> {
-        let base_dir = Path::new("./data");
+        info!("Data dir: {:?}", config.data_dir);
+        // Convert relative path to absolute path
+        let base_dir = if config.data_dir.starts_with('/') {
+            Path::new(&config.data_dir).to_path_buf()
+        } else {
+            std::env::current_dir()
+                .map_err(FileError::Io)?
+                .join(&config.data_dir)
+        };
+
         let user_dir = base_dir.join(user_id); // Create the user directory
         let uuid_dir = user_dir.join(uuid.to_string()); // Create the UUID directory under the user directory
 
@@ -190,9 +205,11 @@ impl FileInfo {
         let final_path = uuid_dir.join(file_name);
         info!("Final path: {:?}", final_path);
 
-        // Persist the temporary file to the final path
-        file.persist(&final_path)?;
-        info!("Persisted file to {:?}", final_path);
+        // Copy the temporary file to the final path
+        tokio::fs::copy(file.path(), &final_path)
+            .await
+            .map_err(FileError::Io)?;
+        info!("Copied file to {:?}", final_path);
 
         Ok(final_path)
     }
@@ -308,6 +325,116 @@ mod tests {
         field_data
     }
 
+    #[tokio::test]
+    async fn test_cross_filesystem_file_operations() {
+        // Setup in-memory database for testing
+        let namespace = "test_ns";
+        let database = &Uuid::new_v4().to_string();
+        let db = SurrealDbClient::memory(namespace, database)
+            .await
+            .expect("Failed to start in-memory surrealdb");
+
+        // Create a test file
+        let content = b"This is a test file for cross-filesystem operations";
+        let file_name = "cross_fs_test.txt";
+        let field_data = create_test_file(content, file_name);
+
+        // Create a FileInfo instance with data_dir in /tmp
+        let user_id = "test_user";
+        let config = AppConfig {
+            data_dir: "/tmp/minne_test_data".to_string(), // Using /tmp which is typically on a different filesystem
+            openai_api_key: "test_key".to_string(),
+            surrealdb_address: "test_address".to_string(),
+            surrealdb_username: "test_user".to_string(),
+            surrealdb_password: "test_pass".to_string(),
+            surrealdb_namespace: "test_ns".to_string(),
+            surrealdb_database: "test_db".to_string(),
+        };
+
+        // Test file creation
+        let file_info = FileInfo::new(field_data, &db, user_id, &config)
+            .await
+            .expect("Failed to create file across filesystems");
+
+        // Verify the file exists and has correct content
+        let file_path = Path::new(&file_info.path);
+        assert!(file_path.exists(), "File should exist at {:?}", file_path);
+
+        let file_content = tokio::fs::read_to_string(file_path)
+            .await
+            .expect("Failed to read file content");
+        assert_eq!(file_content, String::from_utf8_lossy(content));
+
+        // Test file reading
+        let retrieved = FileInfo::get_by_id(&file_info.id, &db)
+            .await
+            .expect("Failed to retrieve file info");
+        assert_eq!(retrieved.id, file_info.id);
+        assert_eq!(retrieved.sha256, file_info.sha256);
+
+        // Test file deletion
+        FileInfo::delete_by_id(&file_info.id, &db)
+            .await
+            .expect("Failed to delete file");
+        assert!(!file_path.exists(), "File should be deleted");
+
+        // Clean up the test directory
+        let _ = tokio::fs::remove_dir_all(&config.data_dir).await;
+    }
+
+    #[tokio::test]
+    async fn test_cross_filesystem_duplicate_detection() {
+        // Setup in-memory database for testing
+        let namespace = "test_ns";
+        let database = &Uuid::new_v4().to_string();
+        let db = SurrealDbClient::memory(namespace, database)
+            .await
+            .expect("Failed to start in-memory surrealdb");
+
+        // Create a test file
+        let content = b"This is a test file for cross-filesystem duplicate detection";
+        let file_name = "cross_fs_duplicate.txt";
+        let field_data = create_test_file(content, file_name);
+
+        // Create a FileInfo instance with data_dir in /tmp
+        let user_id = "test_user";
+        let config = AppConfig {
+            data_dir: "/tmp/minne_test_data".to_string(),
+            openai_api_key: "test_key".to_string(),
+            surrealdb_address: "test_address".to_string(),
+            surrealdb_username: "test_user".to_string(),
+            surrealdb_password: "test_pass".to_string(),
+            surrealdb_namespace: "test_ns".to_string(),
+            surrealdb_database: "test_db".to_string(),
+        };
+
+        // Store the original file
+        let original_file_info = FileInfo::new(field_data, &db, user_id, &config)
+            .await
+            .expect("Failed to create original file");
+
+        // Create another file with the same content but different name
+        let duplicate_name = "cross_fs_duplicate_2.txt";
+        let field_data2 = create_test_file(content, duplicate_name);
+
+        // The system should detect it's the same file and return the original FileInfo
+        let duplicate_file_info = FileInfo::new(field_data2, &db, user_id, &config)
+            .await
+            .expect("Failed to process duplicate file");
+
+        // Verify duplicate detection worked
+        assert_eq!(duplicate_file_info.id, original_file_info.id);
+        assert_eq!(duplicate_file_info.sha256, original_file_info.sha256);
+        assert_eq!(duplicate_file_info.file_name, file_name);
+        assert_ne!(duplicate_file_info.file_name, duplicate_name);
+
+        // Clean up
+        FileInfo::delete_by_id(&original_file_info.id, &db)
+            .await
+            .expect("Failed to delete file");
+        let _ = tokio::fs::remove_dir_all(&config.data_dir).await;
+    }
+
     #[tokio::test]
     async fn test_file_creation() {
         // Setup in-memory database for testing
@@ -324,7 +451,16 @@ mod tests {
 
         // Create a FileInfo instance
         let user_id = "test_user";
-        let file_info = FileInfo::new(field_data, &db, user_id).await;
+        let config = AppConfig {
+            data_dir: "./data".to_string(),
+            openai_api_key: "test_key".to_string(),
+            surrealdb_address: "test_address".to_string(),
+            surrealdb_username: "test_user".to_string(),
+            surrealdb_password: "test_pass".to_string(),
+            surrealdb_namespace: "test_ns".to_string(),
+            surrealdb_database: "test_db".to_string(),
+        };
+        let file_info = FileInfo::new(field_data, &db, user_id, &config).await;
 
         // We can't fully test persistence to disk in unit tests,
         // but we can verify the database record was created
@@ -364,8 +500,18 @@ mod tests {
         let file_name = "original.txt";
         let user_id = "test_user";
 
+        let config = AppConfig {
+            data_dir: "./data".to_string(),
+            openai_api_key: "test_key".to_string(),
+            surrealdb_address: "test_address".to_string(),
+            surrealdb_username: "test_user".to_string(),
+            surrealdb_password: "test_pass".to_string(),
+            surrealdb_namespace: "test_ns".to_string(),
+            surrealdb_database: "test_db".to_string(),
+        };
+
         let field_data1 = create_test_file(content, file_name);
-        let original_file_info = FileInfo::new(field_data1, &db, user_id)
+        let original_file_info = FileInfo::new(field_data1, &db, user_id, &config)
             .await
             .expect("Failed to create original file");
 
@@ -374,7 +520,7 @@ mod tests {
         let field_data2 = create_test_file(content, duplicate_name);
 
         // The system should detect it's the same file and return the original FileInfo
-        let duplicate_file_info = FileInfo::new(field_data2, &db, user_id)
+        let duplicate_file_info = FileInfo::new(field_data2, &db, user_id, &config)
             .await
             .expect("Failed to process duplicate file");
 
@@ -645,8 +791,6 @@ mod tests {
         assert_eq!(retrieved_info.file_name, original_file_info.file_name);
         assert_eq!(retrieved_info.path, original_file_info.path);
         assert_eq!(retrieved_info.mime_type, original_file_info.mime_type);
-        // Optionally compare timestamps if precision isn't an issue
-        // assert_eq!(retrieved_info.created_at, original_file_info.created_at);
     }
 
     #[tokio::test]
@@ -674,4 +818,62 @@ mod tests {
             Ok(_) => panic!("Expected an error, but got Ok"),
         }
     }
+
+    #[tokio::test]
+    async fn test_data_directory_configuration() {
+        // Setup in-memory database for testing
+        let namespace = "test_ns";
+        let database = &Uuid::new_v4().to_string();
+        let db = SurrealDbClient::memory(namespace, database)
+            .await
+            .expect("Failed to start in-memory surrealdb");
+
+        // Create a test file
+        let content = b"This is a test file for data directory configuration";
+        let file_name = "data_dir_test.txt";
+        let field_data = create_test_file(content, file_name);
+
+        // Create a FileInfo instance with a custom data directory
+        let user_id = "test_user";
+        let custom_data_dir = "/tmp/minne_custom_data_dir";
+        let config = AppConfig {
+            data_dir: custom_data_dir.to_string(),
+            openai_api_key: "test_key".to_string(),
+            surrealdb_address: "test_address".to_string(),
+            surrealdb_username: "test_user".to_string(),
+            surrealdb_password: "test_pass".to_string(),
+            surrealdb_namespace: "test_ns".to_string(),
+            surrealdb_database: "test_db".to_string(),
+        };
+
+        // Test file creation
+        let file_info = FileInfo::new(field_data, &db, user_id, &config)
+            .await
+            .expect("Failed to create file in custom data directory");
+
+        // Verify the file exists in the correct location
+        let file_path = Path::new(&file_info.path);
+        assert!(file_path.exists(), "File should exist at {:?}", file_path);
+
+        // Verify the file is in the correct data directory
+        assert!(
+            file_path.starts_with(custom_data_dir),
+            "File should be stored in the custom data directory"
+        );
+
+        // Verify the file has the correct content
+        let file_content = tokio::fs::read_to_string(file_path)
+            .await
+            .expect("Failed to read file content");
+        assert_eq!(file_content, String::from_utf8_lossy(content));
+
+        // Test file deletion
+        FileInfo::delete_by_id(&file_info.id, &db)
+            .await
+            .expect("Failed to delete file");
+        assert!(!file_path.exists(), "File should be deleted");
+
+        // Clean up the test directory
+        let _ = tokio::fs::remove_dir_all(custom_data_dir).await;
+    }
 }

common/src/storage/types/ingestion_payload.rs

@@ -7,30 +7,30 @@ use url::Url;
 pub enum IngestionPayload {
     Url {
         url: String,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
     Text {
         text: String,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
     File {
         file_info: FileInfo,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
 }
 
 impl IngestionPayload {
-    /// Creates ingestion payloads from the provided content, instructions, and files.
+    /// Creates ingestion payloads from the provided content, context, and files.
     ///
     /// # Arguments
     /// * `content` - Optional textual content to be ingressed
-    /// * `instructions` - Instructions for processing the ingress content
+    /// * `context` - context for processing the ingress content
     /// * `category` - Category to classify the ingressed content
     /// * `files` - Vector of `FileInfo` objects containing information about uploaded files
     /// * `user_id` - Identifier of the user performing the ingress operation
@@ -40,7 +40,7 @@ impl IngestionPayload {
     ///   (one per file/content type). On failure, returns an `AppError`.
     pub fn create_ingestion_payload(
         content: Option<String>,
-        instructions: String,
+        context: String,
         category: String,
         files: Vec<FileInfo>,
         user_id: &str,
@@ -55,7 +55,7 @@ impl IngestionPayload {
                     info!("Detected URL: {}", url);
                     object_list.push(IngestionPayload::Url {
                         url: url.to_string(),
-                        instructions: instructions.clone(),
+                        context: context.clone(),
                         category: category.clone(),
                         user_id: user_id.into(),
                     });
@@ -65,7 +65,7 @@ impl IngestionPayload {
                         info!("Treating input as plain text");
                         object_list.push(IngestionPayload::Text {
                             text: input_content.to_string(),
-                            instructions: instructions.clone(),
+                            context: context.clone(),
                             category: category.clone(),
                             user_id: user_id.into(),
                         });
@@ -77,7 +77,7 @@ impl IngestionPayload {
         for file in files {
             object_list.push(IngestionPayload::File {
                 file_info: file,
-                instructions: instructions.clone(),
+                context: context.clone(),
                 category: category.clone(),
                 user_id: user_id.into(),
             })
@@ -126,14 +126,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_url() {
         let url = "https://example.com";
-        let instructions = "Process this URL";
+        let context = "Process this URL";
         let category = "websites";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(url.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -144,13 +144,13 @@ mod tests {
         match &result[0] {
             IngestionPayload::Url {
                 url: payload_url,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 // URL parser may normalize the URL by adding a trailing slash
                 assert!(payload_url == &url.to_string() || payload_url == &format!("{}/", url));
-                assert_eq!(payload_instructions, &instructions);
+                assert_eq!(payload_context, &context);
                 assert_eq!(payload_category, &category);
                 assert_eq!(payload_user_id, &user_id);
             }
@@ -161,14 +161,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_text() {
         let text = "This is some text content";
-        let instructions = "Process this text";
+        let context = "Process this text";
         let category = "notes";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(text.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -179,12 +179,12 @@ mod tests {
         match &result[0] {
             IngestionPayload::Text {
                 text: payload_text,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 assert_eq!(payload_text, text);
-                assert_eq!(payload_instructions, instructions);
+                assert_eq!(payload_context, context);
                 assert_eq!(payload_category, category);
                 assert_eq!(payload_user_id, user_id);
             }
@@ -194,7 +194,7 @@ mod tests {
 
     #[test]
     fn test_create_ingestion_payload_with_file() {
-        let instructions = "Process this file";
+        let context = "Process this file";
         let category = "documents";
         let user_id = "user123";
 
@@ -208,7 +208,7 @@ mod tests {
 
         let result = IngestionPayload::create_ingestion_payload(
             None,
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -219,12 +219,12 @@ mod tests {
         match &result[0] {
             IngestionPayload::File {
                 file_info: payload_file_info,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 assert_eq!(payload_file_info.id, file_info.id);
-                assert_eq!(payload_instructions, instructions);
+                assert_eq!(payload_context, context);
                 assert_eq!(payload_category, category);
                 assert_eq!(payload_user_id, user_id);
             }
@@ -235,7 +235,7 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_url_and_file() {
         let url = "https://example.com";
-        let instructions = "Process this data";
+        let context = "Process this data";
         let category = "mixed";
         let user_id = "user123";
 
@@ -249,7 +249,7 @@ mod tests {
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(url.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -283,14 +283,14 @@ mod tests {
 
     #[test]
     fn test_create_ingestion_payload_empty_input() {
-        let instructions = "Process something";
+        let context = "Process something";
         let category = "empty";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             None,
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -308,14 +308,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_empty_text() {
         let text = ""; // Empty text
-        let instructions = "Process this";
+        let context = "Process this";
         let category = "notes";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(text.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,

common/src/storage/types/ingestion_task.rs

@@ -45,12 +45,12 @@ impl IngestionTask {
         content: IngestionPayload,
         user_id: String,
         db: &SurrealDbClient,
-    ) -> Result<(), AppError> {
-        let job = Self::new(content, user_id).await;
+    ) -> Result<IngestionTask, AppError> {
+        let task = Self::new(content, user_id).await;
 
-        db.store_item(job).await?;
+        db.store_item(task.clone()).await?;
 
-        Ok(())
+        Ok(task)
     }
 
     // Update job status
@@ -110,7 +110,7 @@ mod tests {
     fn create_test_payload(user_id: &str) -> IngestionPayload {
         IngestionPayload::Text {
             text: "Test content".to_string(),
-            instructions: "Test instructions".to_string(),
+            context: "Test context".to_string(),
             category: "Test category".to_string(),
             user_id: user_id.to_string(),
         }

common/src/storage/types/system_prompts.rs

@@ -20,7 +20,7 @@ Example response formats:
 "I found relevant information in multiple entries: [explanation...]"
 "I apologize, but the provided context doesn't contain information about [topic]""#;
 
-pub static DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT: &str = r#"You are an AI assistant. You will receive a text content, along with user instructions and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users instructions and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.
+pub static DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT: &str = r#"You are an AI assistant. You will receive a text content, along with user context and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users context and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.
 
 The JSON should have the following structure:
 

common/src/storage/types/text_content.rs

@@ -16,7 +16,7 @@ stored_object!(TextContent, "text_content", {
     text: String,
     file_info: Option<FileInfo>,
     url_info: Option<UrlInfo>,
-    instructions: String,
+    context: Option<String>,
     category: String,
     user_id: String
 });
@@ -24,7 +24,7 @@ stored_object!(TextContent, "text_content", {
 impl TextContent {
     pub fn new(
         text: String,
-        instructions: String,
+        context: Option<String>,
         category: String,
         file_info: Option<FileInfo>,
         url_info: Option<UrlInfo>,
@@ -38,7 +38,7 @@ impl TextContent {
             text,
             file_info,
             url_info,
-            instructions,
+            context,
             category,
             user_id,
         }
@@ -46,7 +46,7 @@ impl TextContent {
 
     pub async fn patch(
         id: &str,
-        instructions: &str,
+        context: &str,
         category: &str,
         text: &str,
         db: &SurrealDbClient,
@@ -55,7 +55,7 @@ impl TextContent {
 
         let _res: Option<Self> = db
             .update((Self::table_name(), id))
-            .patch(PatchOp::replace("/instructions", instructions))
+            .patch(PatchOp::replace("/context", context))
             .patch(PatchOp::replace("/category", category))
             .patch(PatchOp::replace("/text", text))
             .patch(PatchOp::replace("/updated_at", now))
@@ -73,13 +73,13 @@ mod tests {
     async fn test_text_content_creation() {
         // Test basic object creation
         let text = "Test content text".to_string();
-        let instructions = "Test instructions".to_string();
+        let context = "Test context".to_string();
         let category = "Test category".to_string();
         let user_id = "user123".to_string();
 
         let text_content = TextContent::new(
             text.clone(),
-            instructions.clone(),
+            Some(context.clone()),
             category.clone(),
             None,
             None,
@@ -88,7 +88,7 @@ mod tests {
 
         // Check that the fields are set correctly
         assert_eq!(text_content.text, text);
-        assert_eq!(text_content.instructions, instructions);
+        assert_eq!(text_content.context, Some(context));
         assert_eq!(text_content.category, category);
         assert_eq!(text_content.user_id, user_id);
         assert!(text_content.file_info.is_none());
@@ -100,7 +100,7 @@ mod tests {
     async fn test_text_content_with_url() {
         // Test creating with URL
         let text = "Content with URL".to_string();
-        let instructions = "URL instructions".to_string();
+        let context = "URL context".to_string();
         let category = "URL category".to_string();
         let user_id = "user123".to_string();
         let title = "page_title".to_string();
@@ -115,7 +115,7 @@ mod tests {
 
         let text_content = TextContent::new(
             text.clone(),
-            instructions.clone(),
+            Some(context.clone()),
             category.clone(),
             None,
             url_info.clone(),
@@ -137,13 +137,13 @@ mod tests {
 
         // Create initial text content
         let initial_text = "Initial text".to_string();
-        let initial_instructions = "Initial instructions".to_string();
+        let initial_context = "Initial context".to_string();
         let initial_category = "Initial category".to_string();
         let user_id = "user123".to_string();
 
         let text_content = TextContent::new(
             initial_text,
-            initial_instructions,
+            Some(initial_context),
             initial_category,
             None,
             None,
@@ -158,20 +158,14 @@ mod tests {
         assert!(stored.is_some());
 
         // New values for patch
-        let new_instructions = "Updated instructions";
+        let new_context = "Updated context";
         let new_category = "Updated category";
         let new_text = "Updated text content";
 
         // Apply the patch
-        TextContent::patch(
-            &text_content.id,
-            new_instructions,
-            new_category,
-            new_text,
-            &db,
-        )
-        .await
-        .expect("Failed to patch text content");
+        TextContent::patch(&text_content.id, new_context, new_category, new_text, &db)
+            .await
+            .expect("Failed to patch text content");
 
         // Retrieve the updated content
         let updated: Option<TextContent> = db
@@ -183,7 +177,7 @@ mod tests {
         let updated_content = updated.unwrap();
 
         // Verify the updates
-        assert_eq!(updated_content.instructions, new_instructions);
+        assert_eq!(updated_content.context, Some(new_context.to_string()));
         assert_eq!(updated_content.category, new_category);
         assert_eq!(updated_content.text, new_text);
         assert!(updated_content.updated_at > text_content.updated_at);

common/src/utils/config.rs

@@ -9,6 +9,12 @@ pub struct AppConfig {
     pub surrealdb_password: String,
     pub surrealdb_namespace: String,
     pub surrealdb_database: String,
+    // #[serde(default = "default_data_dir")]
+    pub data_dir: String,
+}
+
+fn default_data_dir() -> String {
+    "./data".to_string()
 }
 
 pub fn get_config() -> Result<AppConfig, ConfigError> {

devenv.lock

@@ -3,10 +3,10 @@
     "devenv": {
       "locked": {
         "dir": "src/modules",
-        "lastModified": 1745333575,
+        "lastModified": 1746681099,
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "cd7456e483ca32b22b84a50015666b44217bd64f",
+        "rev": "a7f2ea275621391209fd702f5ddced32dd56a4e2",
         "type": "github"
       },
       "original": {
@@ -40,10 +40,10 @@
         ]
       },
       "locked": {
-        "lastModified": 1742649964,
+        "lastModified": 1746537231,
         "owner": "cachix",
         "repo": "git-hooks.nix",
-        "rev": "dcf5072734cb576d2b0c59b2ac44f5050b5eac82",
+        "rev": "fa466640195d38ec97cf0493d6d6882bc4d14969",
         "type": "github"
       },
       "original": {
@@ -74,10 +74,10 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1744868846,
+        "lastModified": 1746576598,
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "ebe4301cbd8f81c4f8d3244b3632338bbeb6d49c",
+        "rev": "b3582c75c7f21ce0b429898980eddbbf05c68e55",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,117 +1,65 @@
 {
   description = "Minne application flake";
 
-  # Specify the inputs for our flake (nixpkgs for packages, flake-utils for convenience)
   inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; # Or pin to a specific release/commit
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
   };
 
-  # Define the outputs of our flake (packages, apps, shells, etc.)
   outputs = {
     self,
     nixpkgs,
     flake-utils,
   }:
-  # Use flake-utils to generate outputs for common systems (x86_64-linux, aarch64-linux, x86_64-darwin)
     flake-utils.lib.eachDefaultSystem (
       system: let
-        # Get the package set for the current system
         pkgs = nixpkgs.legacyPackages.${system};
 
         # --- Minne Package Definition ---
-        # This is your core rust application build
         minne-pkg = pkgs.rustPlatform.buildRustPackage {
           pname = "minne";
-          version = "0.1.0"; # Consider fetching this from Cargo.toml later
+          version = "0.1.0";
 
-          # Source is the flake's root directory
           src = self;
 
-          # Assuming you switched to crates.io headless_chrome
           cargoLock = {
             lockFile = ./Cargo.lock;
           };
 
+          # Skip tests due to testing fs operations
+          doCheck = false;
+
           nativeBuildInputs = [
             pkgs.pkg-config
             pkgs.rustfmt
-            # pkgs.makeWrapper # For the postInstall hook
+            pkgs.makeWrapper # For the postInstall hook
           ];
           buildInputs = [
             pkgs.openssl
             pkgs.chromium # Runtime dependency for the browser
           ];
 
-          # Wrap the actual executables to provide CHROME_BIN at runtime
+          # Wrap the actual executables to provide CHROME at runtime
           postInstall = let
-            # Define path to nix-provided chromium executable
             chromium_executable = "${pkgs.chromium}/bin/chromium";
           in ''
-            echo "Wrapping binaries in postInstall hook..."
-            ls -l $out/bin # Add this line to debug which binaries are present
             wrapProgram $out/bin/main \
-              --set CHROME_BIN "${chromium_executable}"
+              --set CHROME "${chromium_executable}"
             wrapProgram $out/bin/worker \
-              --set CHROME_BIN "${chromium_executable}"
-            echo "Finished wrapping."
+              --set CHROME "${chromium_executable}"
           '';
 
           meta = with pkgs.lib; {
             description = "Minne Application";
-            license = licenses.mit; # Adjust if needed
-          };
-        };
-
-        # --- Docker Image Definition (using dockerTools) ---
-        minne-docker-image = pkgs.dockerTools.buildImage {
-          name = "minne";
-          tag = minne-pkg.version;
-
-          # Create an environment containing our packages
-          # and copy its contents to the image's root filesystem.
-          copyToRoot = pkgs.buildEnv {
-            name = "minne-env"; # Name for the build environment derivation
-            paths = [
-              minne-pkg # Include our compiled Rust application
-              pkgs.bashInteractive # Include bash for debugging/interaction
-              pkgs.coreutils # Often useful to have basic utils like ls, cat etc.
-              pkgs.cacert # Include CA certificates for TLS/SSL
-            ];
-            # Optional: Add postBuild hook for the buildEnv if needed
-            # postBuild = '' ... '';
-          };
-
-          # Configure the runtime behavior of the Docker image
-          config = {
-            # Cmd can now likely refer to the binary directly in /bin
-            # (buildEnv symlinks the 'main' binary into the profile's bin)
-            Cmd = ["/bin/main"];
-
-            # ExposedPorts = { "8080/tcp" = {}; };
-            WorkingDir = "/data";
-            # Volumes = { "/data" = {}; };
-
-            # PATH might not need explicit setting if things are in /bin,
-            # but setting explicitly can be safer. buildEnv adds its bin path automatically.
-            Env = [
-              # SSL_CERT_FILE is often essential for HTTPS requests
-              "SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt"
-            ];
+            license = licenses.mit;
           };
         };
       in {
-        # --- Flake Outputs ---
-
-        # Packages: Accessible via 'nix build .#minne' or '.#minne-docker'
         packages = {
           minne = minne-pkg;
-          minne-docker = minne-docker-image;
-          # Default package for 'nix build .'
           default = self.packages.${system}.minne;
         };
 
-        # Apps: Accessible via 'nix run .#main' or '.#worker'
         apps = {
           main = flake-utils.lib.mkApp {
             drv = minne-pkg;
@@ -125,7 +73,6 @@
             drv = minne-pkg;
             name = "server";
           };
-          # Default app for 'nix run .'
           default = self.apps.${system}.main;
         };
       }

html-router/src/html_state.rs

@@ -1,6 +1,6 @@
 use common::storage::db::SurrealDbClient;
 use common::utils::template_engine::{ProvidesTemplateEngine, TemplateEngine};
-use common::{create_template_engine, storage::db::ProvidesDb};
+use common::{create_template_engine, storage::db::ProvidesDb, utils::config::AppConfig};
 use std::sync::Arc;
 use tracing::debug;
 
@@ -12,6 +12,7 @@ pub struct HtmlState {
     pub openai_client: Arc<OpenAIClientType>,
     pub templates: Arc<TemplateEngine>,
     pub session_store: Arc<SessionStoreType>,
+    pub config: AppConfig,
 }
 
 impl HtmlState {
@@ -19,6 +20,7 @@ impl HtmlState {
         db: Arc<SurrealDbClient>,
         openai_client: Arc<OpenAIClientType>,
         session_store: Arc<SessionStoreType>,
+        config: AppConfig,
     ) -> Result<Self, Box<dyn std::error::Error>> {
         let template_engine = create_template_engine!("templates");
         debug!("Template engine created for html_router.");
@@ -28,6 +30,7 @@ impl HtmlState {
             openai_client,
             session_store,
             templates: Arc::new(template_engine),
+            config,
         })
     }
 }

html-router/src/middlewares/response_middleware.rs

@@ -1,7 +1,7 @@
 use axum::{
     extract::State,
     http::{HeaderName, StatusCode},
-    response::{Html, IntoResponse, Response},
+    response::{Html, IntoResponse, Redirect, Response},
     Extension,
 };
 use axum_htmx::{HxRequest, HX_TRIGGER};
@@ -185,7 +185,11 @@ where
                 }
             }
             TemplateKind::Redirect(path) => {
-                (StatusCode::OK, [(axum_htmx::HX_REDIRECT, path.clone())], "").into_response()
+                if is_htmx {
+                    (StatusCode::OK, [(axum_htmx::HX_REDIRECT, path)], "").into_response()
+                } else {
+                    Redirect::to(&path).into_response()
+                }
             }
         }
     } else {

html-router/src/routes/chat/message_response_stream.rs

@@ -8,8 +8,6 @@ use axum::{
         Sse,
     },
 };
-use axum_session_auth::AuthSession;
-use axum_session_surreal::SessionSurrealPool;
 use composite_retrieval::{
     answer_retrieval::{
         create_chat_request, create_user_message_with_history, format_entities_json,
@@ -25,7 +23,6 @@ use json_stream_parser::JsonStreamParser;
 use minijinja::Value;
 use serde::{Deserialize, Serialize};
 use serde_json::from_str;
-use surrealdb::{engine::any::Any, Surreal};
 use tokio::sync::{mpsc::channel, Mutex};
 use tracing::{debug, error};
 
@@ -39,7 +36,7 @@ use common::storage::{
     },
 };
 
-use crate::html_state::HtmlState;
+use crate::{html_state::HtmlState, AuthSessionType};
 
 // Error handling function
 fn create_error_stream(
@@ -110,7 +107,8 @@ pub struct QueryParams {
 
 pub async fn get_response_stream(
     State(state): State<HtmlState>,
-    auth: AuthSession<User, String, SessionSurrealPool<Any>, Surreal<Any>>,
+    auth: AuthSessionType,
+    // auth: AuthSession<User, String, SessionSurrealPool<Any>, Surreal<Any>>,
     Query(params): Query<QueryParams>,
 ) -> Sse<Pin<Box<dyn Stream<Item = Result<Event, axum::Error>> + Send>>> {
     // 1. Authentication and initial data validation

html-router/src/routes/content/handlers.rs

@@ -94,7 +94,7 @@ pub async fn show_text_content_edit_form(
 
 #[derive(Deserialize)]
 pub struct PatchTextContentParams {
-    instructions: String,
+    context: String,
     category: String,
     text: String,
 }
@@ -106,14 +106,7 @@ pub async fn patch_text_content(
 ) -> Result<impl IntoResponse, HtmlError> {
     User::get_and_validate_text_content(&id, &user.id, &state.db).await?;
 
-    TextContent::patch(
-        &id,
-        &form.instructions,
-        &form.category,
-        &form.text,
-        &state.db,
-    )
-    .await?;
+    TextContent::patch(&id, &form.context, &form.category, &form.text, &state.db).await?;
 
     let text_contents = User::get_text_contents(&user.id, &state.db).await?;
     let categories = User::get_user_categories(&user.id, &state.db).await?;
@@ -183,3 +176,24 @@ pub async fn show_content_read_modal(
         TextContentReadModalData { user, text_content },
     ))
 }
+
+pub async fn show_recent_content(
+    State(state): State<HtmlState>,
+    RequireUser(user): RequireUser,
+) -> Result<impl IntoResponse, HtmlError> {
+    let text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
+
+    #[derive(Serialize)]
+    pub struct RecentTextContentData {
+        pub user: User,
+        pub text_contents: Vec<TextContent>,
+    }
+
+    Ok(TemplateResponse::new_template(
+        "/index/signed_in/recent_content.html",
+        RecentTextContentData {
+            user,
+            text_contents,
+        },
+    ))
+}

html-router/src/routes/content/mod.rs

@@ -3,7 +3,7 @@ mod handlers;
 use axum::{extract::FromRef, routing::get, Router};
 use handlers::{
     delete_text_content, patch_text_content, show_content_page, show_content_read_modal,
-    show_text_content_edit_form,
+    show_recent_content, show_text_content_edit_form,
 };
 
 use crate::html_state::HtmlState;
@@ -15,6 +15,7 @@ where
 {
     Router::new()
         .route("/content", get(show_content_page))
+        .route("/content/recent", get(show_recent_content))
         .route("/content/{id}/read", get(show_content_read_modal))
         .route(
             "/content/{id}",

html-router/src/routes/index/handlers.rs

@@ -29,7 +29,7 @@ use crate::html_state::HtmlState;
 #[derive(Serialize)]
 pub struct IndexPageData {
     user: Option<User>,
-    latest_text_contents: Vec<TextContent>,
+    text_contents: Vec<TextContent>,
     active_jobs: Vec<IngestionTask>,
     conversation_archive: Vec<Conversation>,
 }
@@ -39,20 +39,12 @@ pub async fn index_handler(
     auth: AuthSessionType,
 ) -> Result<impl IntoResponse, HtmlError> {
     let Some(user) = auth.current_user else {
-        return Ok(TemplateResponse::new_template(
-            "index/index.html",
-            IndexPageData {
-                user: None,
-                latest_text_contents: vec![],
-                active_jobs: vec![],
-                conversation_archive: vec![],
-            },
-        ));
+        return Ok(TemplateResponse::redirect("/signin"));
     };
 
     let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
 
-    let latest_text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
+    let text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
 
     let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
 
@@ -60,7 +52,7 @@ pub async fn index_handler(
         "index/index.html",
         IndexPageData {
             user: Some(user),
-            latest_text_contents,
+            text_contents,
             active_jobs,
             conversation_archive,
         },

html-router/src/routes/ingestion/handlers.rs

@@ -1,17 +1,27 @@
+use std::{pin::Pin, time::Duration};
+
 use axum::{
-    extract::State,
-    response::{Html, IntoResponse},
+    extract::{Query, State},
+    response::{
+        sse::{Event, KeepAlive},
+        Html, IntoResponse, Sse,
+    },
 };
 use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
-use futures::{future::try_join_all, TryFutureExt};
-use serde::Serialize;
+use futures::{future::try_join_all, stream, Stream, StreamExt, TryFutureExt};
+use minijinja::{context, Value};
+use serde::{Deserialize, Serialize};
 use tempfile::NamedTempFile;
-use tracing::info;
+use tokio::time::sleep;
+use tracing::{error, info};
 
 use common::{
     error::AppError,
     storage::types::{
-        file_info::FileInfo, ingestion_payload::IngestionPayload, ingestion_task::IngestionTask,
+        file_info::FileInfo,
+        ingestion_payload::IngestionPayload,
+        ingestion_task::{IngestionTask, IngestionTaskStatus},
+        text_content::TextContent,
         user::User,
     },
 };
@@ -22,7 +32,7 @@ use crate::{
         auth_middleware::RequireUser,
         response_middleware::{HtmlError, TemplateResponse},
     },
-    routes::index::handlers::ActiveJobsData,
+    AuthSessionType,
 };
 
 pub async fn show_ingress_form(
@@ -54,7 +64,7 @@ pub async fn hide_ingress_form(
 #[derive(Debug, TryFromMultipart)]
 pub struct IngressParams {
     pub content: Option<String>,
-    pub instructions: String,
+    pub context: String,
     pub category: String,
     #[form_data(limit = "10000000")] // Adjust limit as needed
     #[form_data(default)]
@@ -68,7 +78,7 @@ pub async fn process_ingress_form(
 ) -> Result<impl IntoResponse, HtmlError> {
     #[derive(Serialize)]
     pub struct IngressFormData {
-        instructions: String,
+        context: String,
         content: String,
         category: String,
         error: String,
@@ -78,7 +88,7 @@ pub async fn process_ingress_form(
         return Ok(TemplateResponse::new_template(
             "index/signed_in/ingress_form.html",
             IngressFormData {
-                instructions: input.instructions.clone(),
+                context: input.context.clone(),
                 content: input.content.clone().unwrap_or_default(),
                 category: input.category.clone(),
                 error: "You need to either add files or content".to_string(),
@@ -88,17 +98,14 @@ pub async fn process_ingress_form(
 
     info!("{:?}", input);
 
-    let file_infos = try_join_all(
-        input
-            .files
-            .into_iter()
-            .map(|file| FileInfo::new(file, &state.db, &user.id).map_err(AppError::from)),
-    )
+    let file_infos = try_join_all(input.files.into_iter().map(|file| {
+        FileInfo::new(file, &state.db, &user.id, &state.config).map_err(AppError::from)
+    }))
     .await?;
 
     let payloads = IngestionPayload::create_ingestion_payload(
         input.content,
-        input.instructions,
+        input.context,
         input.category,
         file_infos,
         user.id.as_str(),
@@ -106,22 +113,182 @@ pub async fn process_ingress_form(
 
     let futures: Vec<_> = payloads
         .into_iter()
-        .map(|object| {
-            IngestionTask::create_and_add_to_db(object.clone(), user.id.clone(), &state.db)
-        })
+        .map(|object| IngestionTask::create_and_add_to_db(object, user.id.clone(), &state.db))
         .collect();
 
-    try_join_all(futures).await?;
+    let tasks = try_join_all(futures).await?;
 
     // Update the active jobs page with the newly created job
-    let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
+    // let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
 
-    Ok(TemplateResponse::new_partial(
-        "index/signed_in/active_jobs.html",
-        "active_jobs_section",
-        ActiveJobsData {
-            user: user.clone(),
-            active_jobs,
-        },
+    #[derive(Serialize)]
+    struct NewTasksData {
+        user: User,
+        tasks: Vec<IngestionTask>,
+    }
+
+    Ok(TemplateResponse::new_template(
+        "index/signed_in/new_task.html",
+        NewTasksData { user, tasks },
     ))
 }
+
+#[derive(Deserialize)]
+pub struct QueryParams {
+    task_id: String,
+}
+
+// pub async fn get_task_updates_stream(
+//     State(state): State<HtmlState>,
+//     auth: AuthSessionType,
+//     Query(params): Query<QueryParams>,
+// ) -> Sse<Pin<Box<dyn Stream<Item = Result<Event, axum::Error>> + Send>>> {
+//     let task_id = params.task_id.clone();
+//     let db = state.db.clone();
+
+//     let stream = async_stream::stream! {
+//         loop {
+//             match db.get_item::<IngestionTask>(&task_id).await {
+//                 Ok(Some(_task)) => {
+//                     // For now, just sending a placeholder event
+//                     yield Ok(Event::default().event("status").data("hey"));
+//                 },
+//                 _ => {
+//                     yield Ok(Event::default().event("error").data("Failed to get item"));
+//                     break;
+//                 }
+//             }
+//             sleep(Duration::from_secs(5)).await;
+//         }
+//     };
+
+//     Sse::new(stream.boxed()).keep_alive(
+//         KeepAlive::new()
+//             .interval(Duration::from_secs(15))
+//             .text("keep-alive"),
+//     )
+// }
+// Error handling function
+fn create_error_stream(
+    message: impl Into<String>,
+) -> Pin<Box<dyn Stream<Item = Result<Event, axum::Error>> + Send>> {
+    let message = message.into();
+    stream::once(async move { Ok(Event::default().event("error").data(message)) }).boxed()
+}
+
+pub async fn get_task_updates_stream(
+    State(state): State<HtmlState>,
+    auth: AuthSessionType,
+    Query(params): Query<QueryParams>,
+) -> Sse<Pin<Box<dyn Stream<Item = Result<Event, axum::Error>> + Send>>> {
+    let task_id = params.task_id.clone();
+    let db = state.db.clone();
+
+    // 1. Check for authenticated user
+    let current_user = match auth.current_user {
+        Some(user) => user,
+        None => {
+            return Sse::new(create_error_stream(
+                "User not authenticated. Please log in.",
+            ));
+        }
+    };
+
+    // 2. Fetch task for initial authorization and to ensure it exists
+    match db.get_item::<IngestionTask>(&task_id).await {
+        Ok(Some(task)) => {
+            // 3. Validate user ownership
+            if task.user_id != current_user.id {
+                return Sse::new(create_error_stream(
+                    "Access denied: You do not have permission to view updates for this task.",
+                ));
+            }
+
+            let sse_stream = async_stream::stream! {
+                let mut consecutive_db_errors = 0;
+                let max_consecutive_db_errors = 3;
+
+                loop {
+                    match db.get_item::<IngestionTask>(&task_id).await {
+                        Ok(Some(updated_task)) => {
+                            consecutive_db_errors = 0; // Reset error count on success
+
+                            // Format the status message based on IngestionTaskStatus
+                            let status_message = match &updated_task.status {
+                                IngestionTaskStatus::Created => "Created".to_string(),
+                                IngestionTaskStatus::InProgress { attempts, .. } => {
+                                    // Following your template's current display
+                                    format!("In progress, attempt {}", attempts)
+                                }
+                                IngestionTaskStatus::Completed => "Completed".to_string(),
+                                IngestionTaskStatus::Error(ref err_msg) => {
+                                    // Providing a user-friendly error message from the status
+                                    format!("Error: {}", err_msg)
+                                }
+                                IngestionTaskStatus::Cancelled => "Cancelled".to_string(),
+                            };
+
+                            yield Ok(Event::default().event("status").data(status_message));
+
+                            // Check for terminal states to close the stream
+                            match updated_task.status {
+                                IngestionTaskStatus::Completed |
+                                IngestionTaskStatus::Error(_) |
+                                IngestionTaskStatus::Cancelled => {
+                                    // Send a specific event that HTMX uses to close the connection
+                                    // Send a event to reload the recent content
+                                    // Send a event to remove the loading indicatior
+                                    let check_icon = state.templates.render("icons/check_icon.html", &context!{}).unwrap_or("Ok".to_string());
+                                    yield Ok(Event::default().event("stop_loading").data(check_icon));
+                                    yield Ok(Event::default().event("update_latest_content").data("Update latest content"));
+                                    yield Ok(Event::default().event("close_stream").data("Stream complete"));
+                                    break; // Exit loop on terminal states
+                                }
+                                _ => {
+                                    // Not a terminal state, continue polling
+                                }
+                            }
+                        },
+                        Ok(None) => {
+                            // Task disappeared after initial fetch
+                            yield Ok(Event::default().event("error").data("Task not found during update polling."));
+                            break;
+                        }
+                        Err(db_err) => {
+                            error!("Database error while fetching task '{}': {:?}", task_id, db_err);
+                            consecutive_db_errors += 1;
+                            yield Ok(Event::default().event("error").data(format!("Temporary error fetching task update (attempt {}).", consecutive_db_errors)));
+
+                            if consecutive_db_errors >= max_consecutive_db_errors {
+                                error!("Max consecutive DB errors reached for task '{}'. Closing stream.", task_id);
+                                yield Ok(Event::default().event("error").data("Persistent error fetching task updates. Stream closed."));
+                                yield Ok(Event::default().event("close_stream").data("Stream complete"));
+                                break;
+                            }
+                        }
+                    }
+                    sleep(Duration::from_secs(2)).await;
+                }
+            };
+
+            Sse::new(sse_stream.boxed()).keep_alive(
+                KeepAlive::new()
+                    .interval(Duration::from_secs(15))
+                    .text("keep-alive-ping"),
+            )
+        }
+        Ok(None) => Sse::new(create_error_stream(format!(
+            "Task with ID '{}' not found.",
+            task_id
+        ))),
+        Err(e) => {
+            error!(
+                "Failed to fetch task '{}' for authorization: {:?}",
+                task_id, e
+            );
+            Sse::new(create_error_stream(
+                "An error occurred while retrieving task details. Please try again later.",
+            ))
+        }
+    }
+}

html-router/src/routes/ingestion/mod.rs

@@ -1,7 +1,9 @@
 mod handlers;
 
 use axum::{extract::FromRef, routing::get, Router};
-use handlers::{hide_ingress_form, process_ingress_form, show_ingress_form};
+use handlers::{
+    get_task_updates_stream, hide_ingress_form, process_ingress_form, show_ingress_form,
+};
 
 use crate::html_state::HtmlState;
 
@@ -15,5 +17,6 @@ where
             "/ingress-form",
             get(show_ingress_form).post(process_ingress_form),
         )
+        .route("/task/status-stream", get(get_task_updates_stream))
         .route("/hide-ingress-form", get(hide_ingress_form))
 }

html-router/templates/auth/signin_form.html

@@ -4,7 +4,7 @@
   }
 </style>
 
-<div class="flex justify-center grow container mx-auto px-4 sm:px-0 sm:max-w-md flex justify-center flex-col">
+<div class="flex justify-center grow container mx-auto px-4  sm:px-0 sm:max-w-md flex-col">
   <h1
     class="text-5xl sm:text-6xl py-4 pt-10 font-bold bg-linear-to-r from-primary to-secondary text-center text-transparent bg-clip-text">
     Minne

html-router/templates/body_base.html

@@ -10,7 +10,7 @@
       <!-- Navbar -->
       {% include "navigation_bar.html" %}
       <!-- Main Content Area -->
-      <main class="flex-1 overflow-y-auto">
+      <main class="flex flex-1 overflow-y-auto">
         {% block main %}{% endblock %}
       </main>
     </div>

html-router/templates/content/content_list.html

@@ -1,36 +1,37 @@
-<div class="grid sm:grid-cols-2 lg:grid-cols-3 gap-4" id="text_content_cards">
+<div class="columns-1 md:columns-2 2xl:columns-3 gap-4" id="text_content_cards">
   {% for text_content in text_contents %}
-  <div class="card min-w-72 bg-base-100 shadow">
+  <div class="card cursor-pointer mb-4 bg-base-100 shadow break-inside-avoid-column"
+    hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML">
     {% if text_content.url_info %}
-    <img class="rounded-t-md overflow-clip" src="/file/{{text_content.url_info.image_id}}" />
+    <figure>
+      <img src="/file/{{text_content.url_info.image_id}}" alt="website screenshot" />
+    </figure>
     {% endif %}
     <div class="card-body">
       <div class="flex justify-between space-x-2">
         <h2 class="card-title truncate">
           {% if text_content.url_info %}
-          <a href="{{ text_content.url_info.url}}">{{text_content.url_info.title}}</a>
+          {{text_content.url_info.title}}
           {% elif text_content.file_info %}
           {{text_content.file_info.file_name}}
           {% else %}
           {{text_content.text}}
           {% endif %}
         </h2>
-        <div class="flex-shrink-0">
-          {% if text_content.url_info %}
-          {% include "icons/globe_icon.html" %}
-          {% elif text_content.file_info %}
-          {% include "icons/document_icon.html" %}
-          {% else %}
-          {% include "icons/chat_icon.html" %}
-          {% endif %}
-        </div>
       </div>
       <div class="flex items-center justify-between">
         <p class="text-xs opacity-60">
           {{ text_content.created_at | datetimeformat(format="short", tz=user.timezone) }}
         </p>
         <div class="badge badge-soft badge-secondary mr-2">{{ text_content.category }}</div>
-        <div class="flex gap-2">
+        <div class="flex gap-2" hx-on:click="event.stopPropagation()">
+          {% if text_content.url_info %}
+          <button class="btn-btn-square btn-ghost btn-sm">
+            <a href="{{text_content.url_info.url}}" target="_blank" rel="noopener noreferrer">
+              {% include "icons/globe_icon.html" %}
+            </a>
+          </button>
+          {% endif %}
           <button hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML"
             class="btn btn-square btn-ghost btn-sm">
             {% include "icons/read_icon.html" %}

html-router/templates/content/edit_text_content_modal.html

@@ -15,8 +15,8 @@ class="flex flex-col flex-1 h-full"
 <h3 class="text-lg font-bold">Edit Content</h3>
 <div class="form-control">
   <label class="floating-label">
-    <span class="label-text">Instructions</span>
-    <input type="text" name="instructions" value="{{ text_content.instructions }}" class="w-full input input-bordered">
+    <span class="label-text">Context</span>
+    <input type="text" name="context" value="{{ text_content.context }}" class="w-full input input-bordered">
   </label>
 </div>
 <div class="form-control">

html-router/templates/index/index.html

@@ -1,8 +1,11 @@
 {% extends "body_base.html" %}
+
+{% block title %}Minne Dashboard{% endblock %}
+
+{% block head %}
+<script src="/assets/htmx-ext-sse.js" defer></script>
+{% endblock %}
+
 {% block main %}
-{% if user %}
 {% include 'index/signed_in/base.html' %}
-{% else %}
-{% include 'auth/signin_form.html' %}
-{% endif %}
 {% endblock %}

html-router/templates/index/signed_in/active_jobs.html

@@ -1,7 +1,7 @@
 {% block active_jobs_section %}
 <ul id="active_jobs_section" class="list">
   <div class="flex justify-center items-center gap-4">
-    <li class="py-4 text-center font-bold tracking-wide">Active Jobs</li>
+    <li class="py-4 text-center font-bold tracking-wide">Active Tasks</li>
     <button class="cursor-pointer scale-75" hx-get="/active-jobs" hx-target="#active_jobs_section" hx-swap="outerHTML">
       {% include "icons/refresh_icon.html" %}
     </button>
@@ -37,9 +37,6 @@
       {{item.content.Text.text}}
       {% endif %}
     </p>
-    <!-- <button class="btn disabled btn-square btn-ghost btn-sm"> -->
-    <!-- {% include "icons/edit_icon.html" %} -->
-    <!-- </button> -->
     <button hx-delete="/jobs/{{item.id}}" hx-target="#active_jobs_section" hx-swap="outerHTML"
       class="btn btn-square btn-ghost btn-sm">
       {% include "icons/delete_icon.html" %}

html-router/templates/index/signed_in/base.html

@@ -1,12 +1,10 @@
-<div class="flex justify-center grow mt-2 sm:mt-4 gap-6">
+<div class="flex justify-center grow mt-2 sm:mt-4">
   <div class="container">
     {% include 'index/signed_in/searchbar.html' %}
 
-    <div class="grid grid-cols-1 md:grid-cols-2 shadow my-10">
-      {% include "index/signed_in/active_jobs.html" %}
+    {% include "index/signed_in/recent_content.html" %}
 
-      {% include "index/signed_in/recent_content.html" %}
-    </div>
+    {% include "index/signed_in/active_jobs.html" %}
 
   </div>
 </div>

html-router/templates/index/signed_in/ingress_modal.html

@@ -7,18 +7,17 @@ enctype="multipart/form-data"
 <h3 class="text-lg font-bold">Add new content</h3>
 <div class="form-control">
   <label class="floating-label">
-    <span>Instructions</span>
-    <textarea name="instructions" class="textarea w-full validator"
-      placeholder="Enter instructions for the AI here, help it understand what its seeing or how it should relate to the database"
-      required>{{ instructions }}</textarea>
-    <div class="validator-hint hidden">Instructions are required</div>
+    <span>Content</span>
+    <textarea name="content" class="textarea input-bordered w-full"
+      placeholder="Enter the content you want to ingest, it can be an URL or a text snippet">{{ content }}</textarea>
   </label>
 </div>
 <div class="form-control">
   <label class="floating-label">
-    <span>Content</span>
-    <textarea name="content" class="textarea input-bordered w-full"
-      placeholder="Enter the content you want to ingress, it can be an URL or a text snippet">{{ content }}</textarea>
+    <span>Context</span>
+    <textarea name="context" class="textarea w-full"
+      placeholder="Enter context for the AI here, help it understand what its seeing or how it should relate to the database">{{
+      context }}</textarea>
   </label>
 </div>
 <div class="form-control">
@@ -46,7 +45,7 @@ enctype="multipart/form-data"
     const targetElement = document.getElementById('active_jobs_section');
     if (targetElement) {
       form.setAttribute('hx-target', '#active_jobs_section');
-      form.setAttribute('hx-swap', 'outerHTML');
+      form.setAttribute('hx-swap', 'beforeend');
     } else {
       form.setAttribute('hx-swap', 'none');
       form.removeAttribute('hx-target');

html-router/templates/index/signed_in/new_task.html

@@ -0,0 +1,32 @@
+{% for task in tasks %}
+<li class="list-row" hx-ext="sse" sse-connect="/task/status-stream?task_id={{task.id}}" sse-close="close_stream">
+  <div class="bg-secondary rounded-box size-10 flex justify-center items-center text-secondary-content"
+    sse-swap="stop_loading" hx-swap="innerHTML">
+    <span class="loading loading-spinner loading-xl"></span>
+  </div>
+  <div>
+    <div class="flex gap-1">
+      <div sse-swap="status" hx-swap="innerHTML">
+        Created
+      </div>
+      <div hx-get="/content/recent" hx-target="#latest_content_section" hx-swap="outerHTML"
+        hx-trigger="sse:update_latest_content"></div>
+    </div>
+    <div class="text-xs font-semibold opacity-60">
+      {{task.created_at|datetimeformat(format="short", tz=user.timezone)}} </div>
+  </div>
+  <p class="list-col-wrap text-xs [&:before]:content-['Content:_']  [&:before]:opacity-60">
+    {% if task.content.Url %}
+    {{task.content.Url.url}}
+    {% elif task.content.File %}
+    {{task.content.File.file_info.file_name}}
+    {% else %}
+    {{task.content.Text.text}}
+    {% endif %}
+  </p>
+  <button hx-delete="/jobs/{{task.id}}" hx-target="#active_jobs_section" hx-swap="outerHTML"
+    class="btn btn-square btn-ghost btn-sm">
+    {% include "icons/delete_icon.html" %}
+  </button>
+</li>
+{% endfor %}

html-router/templates/index/signed_in/recent_content.html

@@ -1,44 +1,6 @@
 {% block latest_content_section %}
-<ul id="latest_content_section" class="list">
-  <li class="py-4 text-center font-bold tracking-wide">Recently added content</li>
-  {% for item in latest_text_contents %}
-  <li class="list-row">
-    <div class="bg-accent rounded-box size-10 flex justify-center items-center text-accent-content">
-      {% if item.url_info %}
-      {% include "icons/globe_icon.html" %}
-      {% elif item.file_info %}
-      {% include "icons/document_icon.html" %}
-      {% else %}
-      {% include "icons/chat_icon.html" %}
-      {% endif %}
-    </div>
-    <div>
-      <div class="truncate max-w-[160px]">
-        {% if item.url_info %}
-        {{item.url_info.title}}
-        {% elif item.file_info%}
-        {{item.file_info.file_name}}
-        {% else %}
-        {{item.text}}
-        {% endif %}
-      </div>
-      <div class="text-xs font-semibold opacity-60">
-        {{item.created_at|datetimeformat(format="short", tz=user.timezone)}}
-        <span class="badge badge-xs badge-accent ml-1">{{item.category}}</span>
-      </div>
-    </div>
-    <p class="list-col-wrap text-xs [&:before]:content-['Instructions:_']  [&:before]:opacity-60">
-      {{item.instructions}}
-    </p>
-    <button class="btn btn-disabled btn-square btn-ghost btn-sm">
-      {% include "icons/edit_icon.html" %}
-    </button>
-    <button hx-delete="/text-content/{{item.id}}" hx-target="#latest_content_section" hx-swap="outerHTML"
-      class="btn btn-square btn-ghost btn-sm">
-      {% include "icons/delete_icon.html" %}
-
-    </button>
-  </li>
-  {% endfor %}
-</ul>
+<div id="latest_content_section" class="list">
+  <h2 class="font-extrabold">Recent content</h2>
+  {% include "content/content_list.html" %}
+</div>
 {% endblock %}

html-router/templates/navigation_bar.html

@@ -1,30 +1,15 @@
 <nav class="bg-base-200 sticky top-0 z-10">
   <div class="container mx-auto navbar">
-    <div class="flex-none lg:hidden">
-      <label for="my-drawer" aria-label="open sidebar" class="hover:cursor-pointer ">
-        {% include "icons/hamburger_icon.html" %}
-      </label>
-    </div>
     <div class="flex-1">
-      <a class="text-2xl p-2 text-primary font-bold" href="/" hx-boost="true">Minne</a>
+      <a class="text-2xl text-primary font-bold" href="/" hx-boost="true">Minne</a>
     </div>
     <div class="flex-none">
-      <ul class="menu menu-horizontal px-1 items-center">
+      <ul class="menu menu-horizontal px-2 items-center">
         {% include "theme_toggle.html" %}
-        <li><a hx-boost="true" href="/documentation">Docs</a></li>
         {% if user %}
-        <li>
-          <details>
-            <summary>Account</summary>
-            <ul class="bg-base-100 rounded-t-none p-2 z-50 shadow w-40">
-              <li><a hx-boost="true" href="/account">Account</a></li>
-              {% if user.admin %}
-              <li><a hx-boost="true" href="/admin">Admin</a></li>
-              {% endif %}
-              <li><a hx-boost="true" href="/signout">Sign out</a></li>
-            </ul>
-          </details>
-        </li>
+        <label for="my-drawer" aria-label="open sidebar" class="hover:cursor-pointer lg:hidden">
+          {% include "icons/hamburger_icon.html" %}
+        </label>
         {% else %}
         <li><a hx-boost="true" href="/signin">Sign in</a></li>
         {% endif %}

html-router/templates/sidebar.html

@@ -13,11 +13,10 @@
 <div class="drawer-side z-20">
   <label for="my-drawer" aria-label="close sidebar" class="drawer-overlay"></label>
 
-  <ul class="menu p-0 w-64 h-full bg-base-200 text-base-content flex flex-col">
+  <ul class="menu p-0 w-72 h-full bg-base-200 text-base-content flex flex-col">
 
     <!-- === TOP FIXED SECTION === -->
-    <div class="divider mt-14"></div>
-    <div class="px-2">
+    <div class="px-2 mt-14">
       {% for url, name, label in [
       ("/", "home", "Dashboard"),
       ("/knowledge", "book", "Knowledge"),

ingestion-pipeline/src/enricher.rs

@@ -40,17 +40,17 @@ impl IngestionEnricher {
     pub async fn analyze_content(
         &self,
         category: &str,
-        instructions: &str,
+        context: Option<&str>,
         text: &str,
         user_id: &str,
     ) -> Result<LLMEnrichmentResult, AppError> {
         info!("getting similar entitities");
         let similar_entities = self
-            .find_similar_entities(category, instructions, text, user_id)
+            .find_similar_entities(category, context, text, user_id)
             .await?;
         info!("got similar entitities");
         let llm_request = self
-            .prepare_llm_request(category, instructions, text, &similar_entities)
+            .prepare_llm_request(category, context, text, &similar_entities)
             .await?;
         self.perform_analysis(llm_request).await
     }
@@ -58,13 +58,13 @@ impl IngestionEnricher {
     async fn find_similar_entities(
         &self,
         category: &str,
-        instructions: &str,
+        context: Option<&str>,
         text: &str,
         user_id: &str,
     ) -> Result<Vec<KnowledgeEntity>, AppError> {
         let input_text = format!(
-            "content: {}, category: {}, user_instructions: {}",
-            text, category, instructions
+            "content: {}, category: {}, user_context: {:?}",
+            text, category, context
         );
 
         retrieve_entities(&self.db_client, &self.openai_client, &input_text, user_id).await
@@ -73,7 +73,7 @@ impl IngestionEnricher {
     async fn prepare_llm_request(
         &self,
         category: &str,
-        instructions: &str,
+        context: Option<&str>,
         text: &str,
         similar_entities: &[KnowledgeEntity],
     ) -> Result<CreateChatCompletionRequest, AppError> {
@@ -93,8 +93,8 @@ impl IngestionEnricher {
             .collect::<Vec<_>>());
 
         let user_message = format!(
-            "Category:\n{}\nInstructions:\n{}\nContent:\n{}\nExisting KnowledgeEntities in database:\n{}",
-            category, instructions, text, entities_json
+            "Category:\n{}\ncontext:\n{:?}\nContent:\n{}\nExisting KnowledgeEntities in database:\n{}",
+            category, context, text, entities_json
         );
 
         debug!("Prepared LLM request message: {}", user_message);

ingestion-pipeline/src/pipeline.rs

@@ -16,7 +16,7 @@ use common::{
             text_content::TextContent,
         },
     },
-    utils::embedding::generate_embedding,
+    utils::{config::AppConfig, embedding::generate_embedding},
 };
 
 use crate::{
@@ -27,14 +27,20 @@ use crate::{
 pub struct IngestionPipeline {
     db: Arc<SurrealDbClient>,
     openai_client: Arc<async_openai::Client<async_openai::config::OpenAIConfig>>,
+    config: AppConfig,
 }
 
 impl IngestionPipeline {
     pub async fn new(
         db: Arc<SurrealDbClient>,
         openai_client: Arc<async_openai::Client<async_openai::config::OpenAIConfig>>,
+        config: AppConfig,
     ) -> Result<Self, AppError> {
-        Ok(Self { db, openai_client })
+        Ok(Self {
+            db,
+            openai_client,
+            config,
+        })
     }
     pub async fn process_task(&self, task: IngestionTask) -> Result<(), AppError> {
         let current_attempts = match task.status {
@@ -53,7 +59,7 @@ impl IngestionPipeline {
         )
         .await?;
 
-        let text_content = to_text_content(task.content, &self.db).await?;
+        let text_content = to_text_content(task.content, &self.db, &self.config).await?;
 
         match self.process(&text_content).await {
             Ok(_) => {
@@ -113,7 +119,7 @@ impl IngestionPipeline {
         analyser
             .analyze_content(
                 &content.category,
-                &content.instructions,
+                content.context.as_deref(),
                 &content.text,
                 &content.user_id,
             )

ingestion-pipeline/src/types/mod.rs

@@ -14,9 +14,10 @@ use common::{
         ingestion_payload::IngestionPayload,
         text_content::{TextContent, UrlInfo},
     },
+    utils::config::AppConfig,
 };
 use dom_smoothie::{Article, Readability, TextMode};
-use headless_chrome::{Browser, LaunchOptionsBuilder};
+use headless_chrome::Browser;
 use std::io::{Seek, SeekFrom};
 use tempfile::NamedTempFile;
 use tracing::{error, info};
@@ -24,18 +25,19 @@ use tracing::{error, info};
 pub async fn to_text_content(
     ingestion_payload: IngestionPayload,
     db: &SurrealDbClient,
+    config: &AppConfig,
 ) -> Result<TextContent, AppError> {
     match ingestion_payload {
         IngestionPayload::Url {
             url,
-            instructions,
+            context,
             category,
             user_id,
         } => {
-            let (article, file_info) = fetch_article_from_url(&url, db, &user_id).await?;
+            let (article, file_info) = fetch_article_from_url(&url, db, &user_id, &config).await?;
             Ok(TextContent::new(
                 article.text_content.into(),
-                instructions,
+                Some(context),
                 category,
                 None,
                 Some(UrlInfo {
@@ -48,12 +50,12 @@ pub async fn to_text_content(
         }
         IngestionPayload::Text {
             text,
-            instructions,
+            context,
             category,
             user_id,
         } => Ok(TextContent::new(
             text,
-            instructions,
+            Some(context),
             category,
             None,
             None,
@@ -61,14 +63,14 @@ pub async fn to_text_content(
         )),
         IngestionPayload::File {
             file_info,
-            instructions,
+            context,
             category,
             user_id,
         } => {
             let text = extract_text_from_file(&file_info).await?;
             Ok(TextContent::new(
                 text,
-                instructions,
+                Some(context),
                 category,
                 Some(file_info),
                 None,
@@ -101,6 +103,7 @@ async fn fetch_article_from_url(
     url: &str,
     db: &SurrealDbClient,
     user_id: &str,
+    config: &AppConfig,
 ) -> Result<(Article, FileInfo), AppError> {
     info!("Fetching URL: {}", url);
     // Instantiate timer
@@ -110,7 +113,7 @@ async fn fetch_article_from_url(
         #[cfg(feature = "docker")]
         {
             // Use this when compiling for docker
-            let options = LaunchOptionsBuilder::default()
+            let options = headless_chrome::LaunchOptionsBuilder::default()
                 .sandbox(false)
                 .build()
                 .map_err(|e| AppError::InternalError(e.to_string()))?;
@@ -173,7 +176,7 @@ async fn fetch_article_from_url(
     };
 
     // Store screenshot
-    let file_info = FileInfo::new(field_data, db, user_id).await?;
+    let file_info = FileInfo::new(field_data, db, user_id, &config).await?;
 
     // Parse content...
     let config = dom_smoothie::Config {

main/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "main"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2021"
 repository = "https://github.com/perstarkse/minne"
 

main/src/main.rs

@@ -41,10 +41,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
     ));
 
-    let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?;
+    let html_state =
+        HtmlState::new_with_resources(db, openai_client, session_store, config.clone())?;
 
     let api_state = ApiState {
         db: html_state.db.clone(),
+        config: config.clone(),
     };
 
     // Create Axum router
@@ -93,7 +95,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         // Initialize worker components
         let openai_client = Arc::new(async_openai::Client::new());
         let ingestion_pipeline = Arc::new(
-            IngestionPipeline::new(worker_db.clone(), openai_client.clone())
+            IngestionPipeline::new(worker_db.clone(), openai_client.clone(), config.clone())
                 .await
                 .unwrap(),
         );

main/src/server.rs

@@ -39,10 +39,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
     ));
 
-    let html_state = HtmlState::new_with_resources(db, openai_client, session_store)?;
+    let html_state =
+        HtmlState::new_with_resources(db, openai_client, session_store, config.clone())?;
 
     let api_state = ApiState {
         db: html_state.db.clone(),
+        config: config.clone(),
     };
 
     // Create Axum router

main/src/worker.rs

@@ -29,7 +29,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let openai_client = Arc::new(async_openai::Client::new());
 
     let ingestion_pipeline =
-        Arc::new(IngestionPipeline::new(db.clone(), openai_client.clone()).await?);
+        Arc::new(IngestionPipeline::new(db.clone(), openai_client.clone(), config).await?);
 
     run_worker_loop(db, ingestion_pipeline).await
 }

todo.md

@@ -1,43 +1,47 @@
-\[\] allow setting of data storage folder, via envs and config
-\[\] archive ingressed webpage, pdf would be easy
-\[\] full text search
-\[\] rename ingestion instructions to context
-\[\] three js graph explorer
-\[\] three js vector explorer
-\[x\] add user_id to ingress objects
-\[x\] admin controls re registration
-\[x\] change to smoothie dom
-\[x\] chat functionality
-\[x\] chat history
-\[x\] chat styling overhaul
-\[x\] configs primarily get envs
-\[x\] debug why not automatic retrieval of chrome binary works
-\[x\] filtering on categories
-\[x\] fix patch_text_content
-\[x\] gdpr
-\[x\] html ingression
-\[x\] hx-redirect
-\[x\] implement migrations
-\[x\] integrate assets folder in release build
-\[x\] integrate templates in release build
-\[x\] ios shortcut generation
-\[x\] job queue
-\[x\] link to ingressed urls or archives
-\[x\] macro for pagedata?
-\[x\] make sure error messages render correctly
-\[x\] markdown rendering in client
-\[x\] on updates of knowledgeentity create new embeddings
-\[x\] openai api key in config
-\[x\] option to set models, query and processing
-\[x\] page screenshot?
-\[x\] redirects
-\[x\] restrict retrieval to users own objects
-\[x\] smoothie_dom test
-\[x\] store page title
-\[x\] template customization?
-\[x\] templating
-\[x\] testing core functions
-\[x\] user id to fileinfo and data path?
-\[x\] view content
-\[x\] view graph map
-\[x\] view latest
+[] archive ingressed webpage, pdf would be easy
+[] embed surrealdb for the main binary
+[] full text search
+[] three js graph explorer
+[] three js vector explorer
+[x] add user_id to ingress objects
+[x] admin controls re registration
+[x] allow setting of data storage folder, via envs and config
+[x] build docker container on release plan
+[x] change to smoothie dom
+[x] chat functionality
+[x] chat history
+[x] chat styling overhaul
+[x] configs primarily get envs
+[x] debug why not automatic retrieval of chrome binary works
+[x] filtering on categories
+[x] fix card image in content
+[x] fix patch_text_content
+[x] fix redirect for non hx
+[x] html ingression
+[x] hx-redirect
+[x] implement migrations
+[x] integrate assets folder in release build
+[x] integrate templates in release build
+[x] ios shortcut generation
+[x] job queue
+[x] link to ingressed urls or archives
+[x] macro for pagedata?
+[x] make sure error messages render correctly
+[x] markdown rendering in client
+[x] on updates of knowledgeentity create new embeddings
+[x] openai api key in config
+[x] option to set models, query and processing
+[x] page screenshot?
+[x] redirects
+[x] rename ingestion instructions to context
+[x] restrict retrieval to users own objects
+[x] smoothie_dom test
+[x] sse ingestion updates
+[x] store page title
+[x] template customization?
+[x] templating
+[x] testing core functions
+[x] user id to fileinfo and data path?
+[x] view content
+[x] view graph map
+[x] view latest