release: 0.1.4

now supports sending of images as content

remember version bump

fix: typo in readme

updated windows build machine in gh actions

Cargo.lock

@@ -1242,6 +1242,7 @@ dependencies = [
  "tempfile",
  "thiserror 1.0.69",
  "tokio",
+ "tokio-retry",
  "tracing",
  "url",
  "uuid",
@@ -2818,6 +2819,7 @@ dependencies = [
  "async-openai",
  "axum",
  "axum_typed_multipart",
+ "base64 0.22.1",
  "chrono",
  "common",
  "composite-retrieval",
@@ -3126,7 +3128,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 
 [[package]]
 name = "main"
-version = "0.1.1"
+version = "0.1.4"
 dependencies = [
  "anyhow",
  "api-router",
@@ -5793,6 +5795,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-retry"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
+dependencies = [
+ "pin-project",
+ "rand 0.8.5",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.2"

Cargo.toml

@@ -52,6 +52,8 @@ tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 url = { version = "2.5.2", features = ["serde"] }
 uuid = { version = "1.10.0", features = ["v4", "serde"] }
+tokio-retry = "0.3.0"
+base64 = "0.22.1"
 
 [profile.dist]
 inherits = "release"

README.md

@@ -6,28 +6,36 @@
 [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
 [![Latest Release](https://img.shields.io/github/v/release/perstarkse/minne?sort=semver)](https://github.com/perstarkse/minne/releases/latest)
 
+![Screenshot](screenshot.png)
+
+## Demo deployment
+
+To test *Minne* out, enter [this](https://minne-demo.stark.pub) read-only demo deployment to view and test functionality out.
+
 ## The "Why" Behind Minne
 
 For a while I've been fascinated by Zettelkasten-style PKM systems. While tools like Logseq and Obsidian are excellent, I found the manual linking process to be a hindrance for me. I also wanted a centralized storage and easy access across devices.
 
-While developing Minne, I discovered [KaraKeep](https://karakeep.com/) (formerly Hoarder), which is an excellent application in a similar space – you probably want to check it out! However, if you're specifically interested in a PKM that leverages a **SurrealDB as its backend, utilizing both vector and graph retrieval**, offers the **possibility to chat with your knowledge resource**, and provides a blend of manual and AI-driven organization, then Minne might be for you.
+While developing Minne, I discovered [KaraKeep](https://karakeep.com/) (formerly Hoarder), which is an excellent application in a similar space – you probably want to check it out! However, if you're interested in a PKM that builds an automatic network between related concepts using AI, offers search and the **possibility to chat with your knowledge resource**, and provides a blend of manual and AI-driven organization, then Minne might be worth testing.
 
 ## Core Philosophy & Features
 
-Minne is designed to make it incredibly easy to save snippets of text, URLs, and other content(coming if there is demand). Simply send content along with a category tag. Minne then ingests this, leveraging AI to create relevant nodes and relationships within its graph database, alongside your manual categorization. This graph backend, powered by SurrealDB, allows for discoverable connections between your pieces of knowledge.
+Minne is designed to make it incredibly easy to save snippets of text, URLs, and other content (limited, pending demand). Simply send content along with a category tag. Minne then ingests this, leveraging AI to create relevant nodes and relationships within its graph database, alongside your manual categorization. This graph backend allows for discoverable connections between your pieces of knowledge.
 
-You can converse with your knowledge base through an LLM-powered chat interface (via OpenAI API). For those who like to see the bigger picture, Minne also includes an **experimental feature to visually explore your knowledge graph.**
+You can converse with your knowledge base through an LLM-powered chat interface (via OpenAI compatible API, like Ollama or others). For those who like to see the bigger picture, Minne also includes an **experimental feature to visually explore your knowledge graph.**
 
-The application is built for speed and efficiency using Rust with a Server-Side Rendered (SSR) frontend (HTMX and minimal JavaScript). It's fully responsive, offering a complete mobile interface for reading, editing, and managing your content, including the graph database itself. **PWA (Progressive Web App) support** means you can "install" Minne to your device for a native-like experience. For quick capture on the go, especially on iOS, a [**dedicated Shortcut**](https://www.icloud.com/shortcuts/9aa960600ec14329837ba4169f57a166) makes sending content to your Minne instance a breeze.
+You may switch and choose between models used, and have the possiblity to change the prompts to your liking. There is since release **0.1.3** the option to change embeddings length, making it easy to test another embedding model.
+
+The application is built for speed and efficiency using Rust with a Server-Side Rendered (SSR) frontend (HTMX and minimal JavaScript). It's fully responsive, offering a complete mobile interface for reading, editing, and managing your content, including the graph database itself. **PWA (Progressive Web App) support** means you can "install" Minne to your device for a native-like experience. For quick capture on the go on iOS, a [**Shortcut**](https://www.icloud.com/shortcuts/9aa960600ec14329837ba4169f57a166) makes sending content to your Minne instance a breeze.
 
 Minne is open source (AGPL), self-hostable, and can be deployed flexibly: via Nix, Docker Compose, pre-built binaries, or by building from source. It can run as a single `main` binary or as separate `server` and `worker` processes for optimized resource allocation.
 
 ## Tech Stack
 
-- **Backend:** Rust
-- **Frontend:** Server-Side Rendering (SSR) with HTMX, Axum, Minijinja, and plain JavaScript for interactivity.
-- **Database:** SurrealDB (as a graph database)
-- **AI Integration:** OpenAI API (for chat and content processing)
+- **Backend:** Rust. Server-Side Rendering (SSR). Axum. Minijinja for templating.
+- **Frontend:** HTML. HTMX and plain JavaScript for interactivity.
+- **Database:** SurrealDB
+- **AI Integration:** OpenAI API compatible endpoint (for chat and content processing), with support for structured outputs.
 - **Web Content Processing:** Relies on a Chromium instance for robust webpage fetching/rendering.
 
 ## Prerequisites
@@ -80,6 +88,8 @@ This is a great way to manage Minne and its SurrealDB dependency together.
          SURREALDB_DATABASE: "minne_db"
          SURREALDB_NAMESPACE: "minne_ns"
          OPENAI_API_KEY: "your_openai_api_key_here" # IMPORTANT: Replace with your actual key
+         #OPENAI_BASE_URL: "your_ollama_address" # Uncomment this and change it to override the default openai base url 
+         HTTP_PORT: 3000
          DATA_DIR: "/data" # Data directory inside the container
          RUST_LOG: "minne=info,tower_http=info" # Example logging level
        volumes:
@@ -128,18 +138,6 @@ This is a great way to manage Minne and its SurrealDB dependency together.
        driver: bridge
    ```
 
-1. Create a `.env` file in the same directory as your `docker-compose.yml` (recommended for sensitive data):
-
-   ```env
-   OPENAI_API_KEY="your_openai_api_key_here"
-   # You can override other environment variables here if needed
-   # e.g., if you want to expose SurrealDB differently or use different credentials.
-   # SURREALDB_USERNAME_MINNE="custom_user" # If changing Minne's access credentials
-   # SURREALDB_PASSWORD_MINNE="custom_pass"
-   ```
-
-   *(If using a `.env` file, ensure variables in `docker-compose.yml`'s `environment` section reference them like `${OPENAI_API_KEY}` or are directly set if not sensitive and common across setups)*
-
 1. Run:
 
    ```bash
@@ -193,13 +191,13 @@ Minne can be configured using environment variables or a `config.yaml` file plac
 - `SURREALDB_DATABASE`: Database name in SurrealDB (e.g., `minne_db`).
 - `SURREALDB_NAMESPACE`: Namespace in SurrealDB (e.g., `minne_ns`).
 - `OPENAI_API_KEY`: Your API key for OpenAI (e.g., `sk-YourActualOpenAIKeyGoesHere`).
-- `DATA_DIR`: Directory to store local data like fetched webpage content (e.g., `./minne_app_data`).
+- `HTTP_PORT`: Port for the Minne server to listen on (Default: `3000`).
 
 **Optional Configuration:**
 
 - `RUST_LOG`: Controls logging level (e.g., `minne=info,tower_http=debug`).
-- `HTTP_PORT`: Port for the Minne server to listen on (Default: `3000`).
-- `CHROME_ADDRESS`: Address of a remote Chrome DevTools Protocol endpoint (e.g., `http://localhost:9222`, if not using local Chromium managed by Minne/Docker/Nix).
+- `DATA_DIR`: Directory to store local data like fetched webpage content (e.g., `./data`).
+- `OPENAI_BASE_URL`: Base URL to a OpenAI API provider, such as Ollama.
 
 **Example `config.yaml`:**
 
@@ -235,13 +233,32 @@ Once Minne is running:
 1. Engage with the chat interface to query your saved content.
 1. Try the experimental visual graph explorer to see connections.
 
+## AI Configuration & Model Selection
+
+Minne relies on an OpenAI-compatible API for processing content, generating graph relationships, and powering the chat feature.
+
+**Environment Variables / `config.yaml` keys:**
+
+- `OPENAI_API_KEY` (required): Your API key for the chosen AI provider.
+- `OPENAI_BASE_URL` (optional): Use this to override the default OpenAI API URL (`https://api.openai.com/v1`). This is essential for using local models via services like Ollama, or other API providers.
+  - **Example for Ollama:** `http://<your-ollama-ip>:11434/v1`
+
+### Changing Models
+
+Once you have configured the `OPENAI_BASE_URL` to point to your desired provider, you can select the specific models Minne should use.
+
+1. Navigate to the `/admin` page in your Minne instance.
+1. The page will list the models available from your configured endpoint. You can select different models for processing content and for chat.
+1. **Important:** For content processing, Minne relies on structured outputs (function calling). The model and provider you select for this task **must** support this feature.
+1. **Embedding Dimensions:** If you change the embedding model, you **must** update the "Embedding Dimensions" setting in the admin panel to match the output dimensions of your new model (e.g., `text-embedding-3-small` uses 1536, `nomic-embed-text` uses 768). Mismatched dimensions will cause errors. Some newer models will accept a dimension argument, and for these setting the dimensions to whatever should work.
+
 ## Roadmap
 
 I've developed Minne primarily for my own use, but having been in the selfhosted space for a long time, and using the efforts by others, I thought I'd share with the community. Feature requests are welcome.
 The roadmap as of now is:
 
 - Handle uploaded images wisely.
-- An updated explorer of the graph database, and potentially the vector space.
+- An updated explorer of the graph database.
 - A TUI frontend which opens your system default editor for improved writing and document management.
 
 ## Contributing

common/Cargo.toml

@@ -38,6 +38,7 @@ sha2 = { workspace = true }
 url = { workspace = true }
 uuid = { workspace = true } 
 surrealdb-migrations = { workspace = true } 
+tokio-retry = { workspace = true }
 
 
 [features]

common/migrations/20250503_215025_initial_setup.surql

@@ -13,6 +13,8 @@ CREATE system_settings:current CONTENT {
     require_email_verification: false,
     query_model: "gpt-4o-mini",
     processing_model: "gpt-4o-mini",
+    embedding_model: "text-embedding-3-small",
+    embedding_dimensions: 1536,
     query_system_prompt: "You are a knowledgeable assistant with access to a specialized knowledge base. You will be provided with relevant knowledge entities from the database as context. Each knowledge entity contains a name, description, and type, representing different concepts, ideas, and information.\nYour task is to:\n1. Carefully analyze the provided knowledge entities in the context\n2. Answer user questions based on this information\n3. Provide clear, concise, and accurate responses\n4. When referencing information, briefly mention which knowledge entity it came from\n5. If the provided context doesn't contain enough information to answer the question confidently, clearly state this\n6. If only partial information is available, explain what you can answer and what information is missing\n7. Avoid making assumptions or providing information not supported by the context\n8. Output the references to the documents. Use the UUIDs and make sure they are correct!\nRemember:\n- Be direct and honest about the limitations of your knowledge\n- Cite the relevant knowledge entities when providing information, but only provide the UUIDs in the reference array\n- If you need to combine information from multiple entities, explain how they connect\n- Don't speculate beyond what's provided in the context\nExample response formats:\n\"Based on [Entity Name], [answer...]\"\n\"I found relevant information in multiple entries: [explanation...]\"\n\"I apologize, but the provided context doesn't contain information about [topic]\"",
     ingestion_system_prompt: "You are an AI assistant. You will receive a text content, along with user context and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users context and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.\nThe JSON should have the following structure:\n{\n\"knowledge_entities\": [\n{\n\"key\": \"unique-key-1\",\n\"name\": \"Entity Name\",\n\"description\": \"A detailed description of the entity.\",\n\"entity_type\": \"TypeOfEntity\"\n},\n// More entities...\n],\n\"relationships\": [\n{\n\"type\": \"RelationshipType\",\n\"source\": \"unique-key-1 or UUID from existing database\",\n\"target\": \"unique-key-1 or UUID from existing database\"\n},\n// More relationships...\n]\n}\nGuidelines:\n1. Do NOT generate any IDs or UUIDs. Use a unique `key` for each knowledge entity.\n2. Each KnowledgeEntity should have a unique `key`, a meaningful `name`, and a descriptive `description`.\n3. Define the type of each KnowledgeEntity using the following categories: Idea, Project, Document, Page, TextSnippet.\n4. Establish relationships between entities using types like RelatedTo, RelevantTo, SimilarTo.\n5. Use the `source` key to indicate the originating entity and the `target` key to indicate the related entity\"\n6. You will be presented with a few existing KnowledgeEntities that are similar to the current ones. They will have an existing UUID. When creating relationships to these entities, use their UUID.\n7. Only create relationships between existing KnowledgeEntities.\n8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.\n9. A new relationship MUST include a newly created KnowledgeEntity."
 };

common/migrations/20250606_234535_add_embedding_model_and_dimensions_to_system_settings.surql

@@ -0,0 +1,7 @@
+DEFINE FIELD IF NOT EXISTS embedding_model ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS embedding_dimensions ON system_settings TYPE int;
+
+UPDATE system_settings:current SET
+	embedding_model =  "text-embedding-3-small",
+    embedding_dimensions = 1536
+WHERE embedding_model == NONE && embedding_dimensions == NONE;

common/migrations/20250627_120000_add_image_processing_settings.surql

@@ -0,0 +1,7 @@
+DEFINE FIELD IF NOT EXISTS image_processing_model ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS image_processing_prompt ON system_settings TYPE string;
+
+UPDATE system_settings:current SET
+	image_processing_model =  "gpt-4o-mini",
+    image_processing_prompt = "Analyze this image and respond based on its primary content:\n - If the image is mainly text (document, screenshot, sign), transcribe the text verbatim.\n - If the image is mainly visual (photograph, art, landscape), provide a concise description of the scene.\n - For hybrid images (diagrams, ads), briefly describe the visual, then transcribe the text under a Text: heading.\n\n Respond directly with the analysis."
+WHERE image_processing_model == NONE && image_processing_prompt == NONE;

common/migrations/20250627_231035_remove_job_table.surql

@@ -0,0 +1 @@
+REMOVE TABLE job;

common/migrations/definitions/20250514_142342_add_full_text_search_text_content.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -98,7 +98,7 @@\n DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;\n DEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;\n-DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at; # For get_latest_knowledge_entities\n+DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;\n\n # Defines the schema for the 'message' table.\n\n@@ -157,6 +157,8 @@\n DEFINE FIELD IF NOT EXISTS require_email_verification ON system_settings TYPE bool;\n DEFINE FIELD IF NOT EXISTS query_model ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS processing_model ON system_settings TYPE string;\n+DEFINE FIELD IF NOT EXISTS embedding_model ON system_settings TYPE string;\n+DEFINE FIELD IF NOT EXISTS embedding_dimensions ON system_settings TYPE int;\n DEFINE FIELD IF NOT EXISTS query_system_prompt ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS ingestion_system_prompt ON system_settings TYPE string;\n\n","events":null}

common/migrations/definitions/20250606_234535_add_embedding_model_and_dimensions_to_system_settings.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -51,23 +51,23 @@\n\n # Defines the schema for the 'ingestion_task' table (used by IngestionTask).\n\n-DEFINE TABLE IF NOT EXISTS job SCHEMALESS;\n+DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON job TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON job TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n\n # Custom fields from the IngestionTask struct\n # IngestionPayload is complex, store as object\n-DEFINE FIELD IF NOT EXISTS content ON job TYPE object;\n+DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n # IngestionTaskStatus can hold data (InProgress), store as object\n-DEFINE FIELD IF NOT EXISTS status ON job TYPE object;\n-DEFINE FIELD IF NOT EXISTS user_id ON job TYPE string;\n+DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n+DEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;\n\n # Indexes explicitly defined in build_indexes and useful for get_unfinished_tasks\n-DEFINE INDEX IF NOT EXISTS idx_job_status ON job FIELDS status;\n-DEFINE INDEX IF NOT EXISTS idx_job_user ON job FIELDS user_id;\n-DEFINE INDEX IF NOT EXISTS idx_job_created ON job FIELDS created_at;\n+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_status ON ingestion_task FIELDS status;\n+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_user ON ingestion_task FIELDS user_id;\n+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_created ON ingestion_task FIELDS created_at;\n\n # Defines the schema for the 'knowledge_entity' table.\n\n","events":null}

common/migrations/definitions/20250627_120000_add_image_processing_settings.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -57,10 +57,7 @@\n DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n\n-# Custom fields from the IngestionTask struct\n-# IngestionPayload is complex, store as object\n DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n-# IngestionTaskStatus can hold data (InProgress), store as object\n DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n DEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;\n\n@@ -157,10 +154,12 @@\n DEFINE FIELD IF NOT EXISTS require_email_verification ON system_settings TYPE bool;\n DEFINE FIELD IF NOT EXISTS query_model ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS processing_model ON system_settings TYPE string;\n+DEFINE FIELD IF NOT EXISTS image_processing_model ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS embedding_model ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS embedding_dimensions ON system_settings TYPE int;\n DEFINE FIELD IF NOT EXISTS query_system_prompt ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS ingestion_system_prompt ON system_settings TYPE string;\n+DEFINE FIELD IF NOT EXISTS image_processing_prompt ON system_settings TYPE string;\n\n # Defines the schema for the 'text_chunk' table.\n\n","events":null}

common/schemas/ingestion_task.surql

@@ -1,19 +1,16 @@
 # Defines the schema for the 'ingestion_task' table (used by IngestionTask).
 
-DEFINE TABLE IF NOT EXISTS job SCHEMALESS;
+DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;
 
 # Standard fields
-DEFINE FIELD IF NOT EXISTS created_at ON job TYPE string;
-DEFINE FIELD IF NOT EXISTS updated_at ON job TYPE string;
+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;
+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;
 
-# Custom fields from the IngestionTask struct
-# IngestionPayload is complex, store as object
-DEFINE FIELD IF NOT EXISTS content ON job TYPE object;
-# IngestionTaskStatus can hold data (InProgress), store as object
-DEFINE FIELD IF NOT EXISTS status ON job TYPE object;
-DEFINE FIELD IF NOT EXISTS user_id ON job TYPE string;
+DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;
+DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;
+DEFINE FIELD IF NOT EXISTS user_id ON ingestion_task TYPE string;
 
 # Indexes explicitly defined in build_indexes and useful for get_unfinished_tasks
-DEFINE INDEX IF NOT EXISTS idx_job_status ON job FIELDS status;
-DEFINE INDEX IF NOT EXISTS idx_job_user ON job FIELDS user_id;
-DEFINE INDEX IF NOT EXISTS idx_job_created ON job FIELDS created_at;
+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_status ON ingestion_task FIELDS status;
+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_user ON ingestion_task FIELDS user_id;
+DEFINE INDEX IF NOT EXISTS idx_ingestion_task_created ON ingestion_task FIELDS created_at;

common/schemas/knowledge_entity.surql

@@ -27,4 +27,4 @@ DEFINE INDEX IF NOT EXISTS idx_embedding_entities ON knowledge_entity FIELDS emb
 DEFINE INDEX IF NOT EXISTS knowledge_entity_user_id_idx ON knowledge_entity FIELDS user_id;
 DEFINE INDEX IF NOT EXISTS knowledge_entity_source_id_idx ON knowledge_entity FIELDS source_id;
 DEFINE INDEX IF NOT EXISTS knowledge_entity_entity_type_idx ON knowledge_entity FIELDS entity_type;
-DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at; # For get_latest_knowledge_entities
+DEFINE INDEX IF NOT EXISTS knowledge_entity_created_at_idx ON knowledge_entity FIELDS created_at;

common/schemas/system_settings.surql

@@ -7,5 +7,9 @@ DEFINE FIELD IF NOT EXISTS registrations_enabled ON system_settings TYPE bool;
 DEFINE FIELD IF NOT EXISTS require_email_verification ON system_settings TYPE bool;
 DEFINE FIELD IF NOT EXISTS query_model ON system_settings TYPE string;
 DEFINE FIELD IF NOT EXISTS processing_model ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS image_processing_model ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS embedding_model ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS embedding_dimensions ON system_settings TYPE int;
 DEFINE FIELD IF NOT EXISTS query_system_prompt ON system_settings TYPE string;
 DEFINE FIELD IF NOT EXISTS ingestion_system_prompt ON system_settings TYPE string;
+DEFINE FIELD IF NOT EXISTS image_processing_prompt ON system_settings TYPE string;

common/src/error.rs

@@ -29,8 +29,8 @@ pub enum AppError {
     Io(#[from] std::io::Error),
     #[error("Reqwest error: {0}")]
     Reqwest(#[from] reqwest::Error),
-    #[error("Tiktoken error: {0}")]
-    Tiktoken(#[from] anyhow::Error),
+    #[error("Anyhow error: {0}")]
+    Anyhow(#[from] anyhow::Error),
     #[error("Ingestion Processing error: {0}")]
     Processing(String),
     #[error("DOM smoothie error: {0}")]

common/src/storage/db.rs

@@ -11,6 +11,7 @@ use surrealdb::{
     Error, Notification, Surreal,
 };
 use surrealdb_migrations::MigrationRunner;
+use tracing::debug;
 
 static MIGRATIONS_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/");
 
@@ -50,6 +51,7 @@ impl SurrealDbClient {
     pub async fn create_session_store(
         &self,
     ) -> Result<SessionStore<SessionSurrealPool<Any>>, SessionError> {
+        debug!("Creating session store");
         SessionStore::new(
             Some(self.client.clone().into()),
             SessionConfig::default()
@@ -65,6 +67,7 @@ impl SurrealDbClient {
     /// the database and selecting the appropriate namespace and database, but before
     /// the application starts performing operations that rely on the schema.
     pub async fn apply_migrations(&self) -> Result<(), AppError> {
+        debug!("Applying migrations");
         MigrationRunner::new(&self.client)
             .load_files(&MIGRATIONS_DIR)
             .up()
@@ -76,6 +79,7 @@ impl SurrealDbClient {
 
     /// Operation to rebuild indexes
     pub async fn rebuild_indexes(&self) -> Result<(), Error> {
+        debug!("Rebuilding indexes");
         self.client
             .query("REBUILD INDEX IF EXISTS idx_embedding_chunks ON text_chunk")
             .await?;

common/src/storage/types/file_info.rs

@@ -349,6 +349,8 @@ mod tests {
             surrealdb_password: "test_pass".to_string(),
             surrealdb_namespace: "test_ns".to_string(),
             surrealdb_database: "test_db".to_string(),
+            http_port: 3000,
+            openai_base_url: "..".to_string(),
         };
 
         // Test file creation
@@ -406,6 +408,8 @@ mod tests {
             surrealdb_password: "test_pass".to_string(),
             surrealdb_namespace: "test_ns".to_string(),
             surrealdb_database: "test_db".to_string(),
+            http_port: 3000,
+            openai_base_url: "..".to_string(),
         };
 
         // Store the original file
@@ -459,6 +463,8 @@ mod tests {
             surrealdb_password: "test_pass".to_string(),
             surrealdb_namespace: "test_ns".to_string(),
             surrealdb_database: "test_db".to_string(),
+            http_port: 3000,
+            openai_base_url: "..".to_string(),
         };
         let file_info = FileInfo::new(field_data, &db, user_id, &config).await;
 
@@ -508,6 +514,8 @@ mod tests {
             surrealdb_password: "test_pass".to_string(),
             surrealdb_namespace: "test_ns".to_string(),
             surrealdb_database: "test_db".to_string(),
+            http_port: 3000,
+            openai_base_url: "..".to_string(),
         };
 
         let field_data1 = create_test_file(content, file_name);
@@ -844,6 +852,8 @@ mod tests {
             surrealdb_password: "test_pass".to_string(),
             surrealdb_namespace: "test_ns".to_string(),
             surrealdb_database: "test_db".to_string(),
+            http_port: 3000,
+            openai_base_url: "..".to_string(),
         };
 
         // Test file creation

common/src/storage/types/ingestion_task.rs

@@ -7,6 +7,7 @@ use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
 use super::ingestion_payload::IngestionPayload;
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(tag = "name")]
 pub enum IngestionTaskStatus {
     Created,
     InProgress {
@@ -14,7 +15,9 @@ pub enum IngestionTaskStatus {
         last_attempt: DateTime<Utc>,
     },
     Completed,
-    Error(String),
+    Error {
+        message: String,
+    },
     Cancelled,
 }
 
@@ -85,10 +88,10 @@ impl IngestionTask {
             .query(
                 "SELECT * FROM type::table($table) 
              WHERE 
-                status = 'Created' 
+                status.name = 'Created' 
                 OR (
-                    status.InProgress != NONE 
-                    AND status.InProgress.attempts < $max_attempts
+                    status.name = 'InProgress'
+                    AND status.attempts < $max_attempts
                 )
              ORDER BY created_at ASC",
             )
@@ -241,7 +244,9 @@ mod tests {
         completed_task.status = IngestionTaskStatus::Completed;
 
         let mut error_task = IngestionTask::new(payload.clone(), user_id.to_string()).await;
-        error_task.status = IngestionTaskStatus::Error("Test error".to_string());
+        error_task.status = IngestionTaskStatus::Error {
+            message: "Test error".to_string(),
+        };
 
         // Store all tasks
         db.store_item(created_task)
@@ -280,7 +285,7 @@ mod tests {
                     }
                 }
                 IngestionTaskStatus::Completed => "Completed",
-                IngestionTaskStatus::Error(_) => "Error",
+                IngestionTaskStatus::Error { .. } => "Error",
                 IngestionTaskStatus::Cancelled => "Cancelled",
             })
             .collect();

common/src/storage/types/knowledge_entity.rs

@@ -1,8 +1,15 @@
+use std::collections::HashMap;
+
 use crate::{
     error::AppError, storage::db::SurrealDbClient, stored_object,
     utils::embedding::generate_embedding,
 };
 use async_openai::{config::OpenAIConfig, Client};
+use tokio_retry::{
+    strategy::{jitter, ExponentialBackoff},
+    Retry,
+};
+use tracing::{error, info};
 use uuid::Uuid;
 
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
@@ -94,7 +101,7 @@ impl KnowledgeEntity {
             "name: {}, description: {}, type: {:?}",
             name, description, entity_type
         );
-        let embedding = generate_embedding(ai_client, &embedding_input).await?;
+        let embedding = generate_embedding(ai_client, &embedding_input, db_client).await?;
 
         db_client
             .client
@@ -118,6 +125,104 @@ impl KnowledgeEntity {
 
         Ok(())
     }
+
+    /// Re-creates embeddings for all knowledge entities in the database.
+    ///
+    /// This is a costly operation that should be run in the background. It follows the same
+    /// pattern as the text chunk update:
+    /// 1. Re-defines the vector index with the new dimensions.
+    /// 2. Fetches all existing entities.
+    /// 3. Sequentially regenerates the embedding for each and updates the record.
+    pub async fn update_all_embeddings(
+        db: &SurrealDbClient,
+        openai_client: &Client<OpenAIConfig>,
+        new_model: &str,
+        new_dimensions: u32,
+    ) -> Result<(), AppError> {
+        info!(
+            "Starting re-embedding process for all knowledge entities. New dimensions: {}",
+            new_dimensions
+        );
+
+        // Fetch all entities first
+        let all_entities: Vec<KnowledgeEntity> = db.select(Self::table_name()).await?;
+        let total_entities = all_entities.len();
+        if total_entities == 0 {
+            info!("No knowledge entities to update. Skipping.");
+            return Ok(());
+        }
+        info!("Found {} entities to process.", total_entities);
+
+        // Generate all new embeddings in memory
+        let mut new_embeddings: HashMap<String, Vec<f32>> = HashMap::new();
+        info!("Generating new embeddings for all entities...");
+        for entity in all_entities.iter() {
+            let embedding_input = format!(
+                "name: {}, description: {}, type: {:?}",
+                entity.name, entity.description, entity.entity_type
+            );
+            let retry_strategy = ExponentialBackoff::from_millis(100).map(jitter).take(3);
+
+            let embedding = Retry::spawn(retry_strategy, || {
+                crate::utils::embedding::generate_embedding_with_params(
+                    openai_client,
+                    &embedding_input,
+                    new_model,
+                    new_dimensions,
+                )
+            })
+            .await?;
+
+            // Check embedding lengths
+            if embedding.len() != new_dimensions as usize {
+                let err_msg = format!(
+                "CRITICAL: Generated embedding for entity {} has incorrect dimension ({}). Expected {}. Aborting.",
+                entity.id, embedding.len(), new_dimensions
+            );
+                error!("{}", err_msg);
+                return Err(AppError::InternalError(err_msg));
+            }
+            new_embeddings.insert(entity.id.clone(), embedding);
+        }
+        info!("Successfully generated all new embeddings.");
+
+        // Perform DB updates in a single transaction
+        info!("Applying schema and data changes in a transaction...");
+        let mut transaction_query = String::from("BEGIN TRANSACTION;");
+
+        // Add all update statements
+        for (id, embedding) in new_embeddings {
+            // We must properly serialize the vector for the SurrealQL query string
+            let embedding_str = format!(
+                "[{}]",
+                embedding
+                    .iter()
+                    .map(|f| f.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",")
+            );
+            transaction_query.push_str(&format!(
+            "UPDATE type::thing('knowledge_entity', '{}') SET embedding = {}, updated_at = time::now();",
+            id, embedding_str
+        ));
+        }
+
+        // Re-create the index after updating the data that it will index
+        transaction_query
+            .push_str("REMOVE INDEX idx_embedding_entities ON TABLE knowledge_entity;");
+        transaction_query.push_str(&format!(
+        "DEFINE INDEX idx_embedding_entities ON TABLE knowledge_entity FIELDS embedding HNSW DIMENSION {};",
+        new_dimensions
+    ));
+
+        transaction_query.push_str("COMMIT TRANSACTION;");
+
+        // Execute the entire atomic operation
+        db.query(transaction_query).await?;
+
+        info!("Re-embedding process for knowledge entities completed successfully.");
+        Ok(())
+    }
 }
 
 #[cfg(test)]

common/src/storage/types/system_prompts.rs

@@ -54,3 +54,10 @@ Guidelines:
 7. Only create relationships between existing KnowledgeEntities.
 8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.
 9. A new relationship MUST include a newly created KnowledgeEntity."#;
+
+pub static DEFAULT_IMAGE_PROCESSING_PROMPT: &str = r#"Analyze this image and respond based on its primary content:
+- If the image is mainly text (document, screenshot, sign), transcribe the text verbatim.
+- If the image is mainly visual (photograph, art, landscape), provide a concise description of the scene.
+- For hybrid images (diagrams, ads), briefly describe the visual, then transcribe the text under a "Text:" heading.
+
+Respond directly with the analysis."#;

common/src/storage/types/system_settings.rs

@@ -11,8 +11,12 @@ pub struct SystemSettings {
     pub require_email_verification: bool,
     pub query_model: String,
     pub processing_model: String,
+    pub embedding_model: String,
+    pub embedding_dimensions: u32,
     pub query_system_prompt: String,
     pub ingestion_system_prompt: String,
+    pub image_processing_model: String,
+    pub image_processing_prompt: String,
 }
 
 impl StoredObject for SystemSettings {
@@ -44,25 +48,12 @@ impl SystemSettings {
             "Something went wrong updating the settings".into(),
         ))
     }
-
-    pub fn new() -> Self {
-        Self {
-            id: "current".to_string(),
-            query_system_prompt: crate::storage::types::system_prompts::DEFAULT_QUERY_SYSTEM_PROMPT
-                .to_string(),
-            ingestion_system_prompt:
-                crate::storage::types::system_prompts::DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT
-                    .to_string(),
-            query_model: "gpt-4o-mini".to_string(),
-            processing_model: "gpt-4o-mini".to_string(),
-            registrations_enabled: true,
-            require_email_verification: false,
-        }
-    }
 }
 
 #[cfg(test)]
 mod tests {
+    use crate::storage::types::text_chunk::TextChunk;
+
     use super::*;
     use uuid::Uuid;
 
@@ -89,6 +80,7 @@ mod tests {
         assert_eq!(settings.require_email_verification, false);
         assert_eq!(settings.query_model, "gpt-4o-mini");
         assert_eq!(settings.processing_model, "gpt-4o-mini");
+        assert_eq!(settings.image_processing_model, "gpt-4o-mini");
         // Dont test these for now, having a hard time getting the formatting exactly the same
         // assert_eq!(
         //     settings.query_system_prompt,
@@ -157,7 +149,7 @@ mod tests {
             .expect("Failed to apply migrations");
 
         // Create updated settings
-        let mut updated_settings = SystemSettings::new();
+        let mut updated_settings = SystemSettings::get_current(&db).await.unwrap();
         updated_settings.id = "current".to_string();
         updated_settings.registrations_enabled = false;
         updated_settings.require_email_verification = true;
@@ -206,21 +198,60 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_new_method() {
-        let settings = SystemSettings::new();
+    async fn test_migration_after_changing_embedding_length() {
+        let db = SurrealDbClient::memory("test", &Uuid::new_v4().to_string())
+            .await
+            .expect("Failed to start DB");
 
-        assert!(settings.id.len() > 0);
-        assert_eq!(settings.registrations_enabled, true);
-        assert_eq!(settings.require_email_verification, false);
-        assert_eq!(settings.query_model, "gpt-4o-mini");
-        assert_eq!(settings.processing_model, "gpt-4o-mini");
-        assert_eq!(
-            settings.query_system_prompt,
-            crate::storage::types::system_prompts::DEFAULT_QUERY_SYSTEM_PROMPT
-        );
-        assert_eq!(
-            settings.ingestion_system_prompt,
-            crate::storage::types::system_prompts::DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT
+        // Apply initial migrations. This sets up the text_chunk index with DIMENSION 1536.
+        db.apply_migrations()
+            .await
+            .expect("Initial migration failed");
+
+        let initial_chunk = TextChunk::new(
+            "source1".into(),
+            "This chunk has the original dimension".into(),
+            vec![0.1; 1536],
+            "user1".into(),
         );
+
+        db.store_item(initial_chunk.clone())
+            .await
+            .expect("Failed to store initial chunk");
+
+        async fn simulate_reembedding(
+            db: &SurrealDbClient,
+            target_dimension: usize,
+            initial_chunk: TextChunk,
+        ) {
+            db.query("REMOVE INDEX idx_embedding_chunks ON TABLE text_chunk;")
+                .await
+                .unwrap();
+            let define_index_query = format!(
+                         "DEFINE INDEX idx_embedding_chunks ON TABLE text_chunk FIELDS embedding HNSW DIMENSION {};",
+                         target_dimension
+                     );
+            db.query(define_index_query)
+                .await
+                .expect("Re-defining index should succeed");
+
+            let new_embedding = vec![0.5; target_dimension];
+            let sql = "UPDATE type::thing('text_chunk', $id) SET embedding = $embedding;";
+
+            let update_result = db
+                .client
+                .query(sql)
+                .bind(("id", initial_chunk.id.clone()))
+                .bind(("embedding", new_embedding))
+                .await;
+
+            assert!(update_result.is_ok());
+        }
+
+        simulate_reembedding(&db, 768, initial_chunk).await;
+
+        let migration_result = db.apply_migrations().await;
+
+        assert!(migration_result.is_ok(), "Migrations should not fail");
     }
 }

common/src/storage/types/text_chunk.rs

@@ -1,4 +1,13 @@
+use std::collections::HashMap;
+
 use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
+use async_openai::{config::OpenAIConfig, Client};
+use tokio_retry::{
+    strategy::{jitter, ExponentialBackoff},
+    Retry,
+};
+
+use tracing::{error, info};
 use uuid::Uuid;
 
 stored_object!(TextChunk, "text_chunk", {
@@ -35,6 +44,99 @@ impl TextChunk {
 
         Ok(())
     }
+
+    /// Re-creates embeddings for all text chunks using a safe, atomic transaction.
+    ///
+    /// This is a costly operation that should be run in the background. It performs these steps:
+    /// 1. **Fetches All Chunks**: Loads all existing text_chunk records into memory.
+    /// 2. **Generates All Embeddings**: Creates new embeddings for every chunk. If any fails or
+    ///    has the wrong dimension, the entire operation is aborted before any DB changes are made.
+    /// 3. **Executes Atomic Transaction**: All data updates and the index recreation are
+    ///    performed in a single, all-or-nothing database transaction.
+    pub async fn update_all_embeddings(
+        db: &SurrealDbClient,
+        openai_client: &Client<OpenAIConfig>,
+        new_model: &str,
+        new_dimensions: u32,
+    ) -> Result<(), AppError> {
+        info!(
+            "Starting re-embedding process for all text chunks. New dimensions: {}",
+            new_dimensions
+        );
+
+        // Fetch all chunks first
+        let all_chunks: Vec<TextChunk> = db.select(Self::table_name()).await?;
+        let total_chunks = all_chunks.len();
+        if total_chunks == 0 {
+            info!("No text chunks to update. Skipping.");
+            return Ok(());
+        }
+        info!("Found {} chunks to process.", total_chunks);
+
+        // Generate all new embeddings in memory
+        let mut new_embeddings: HashMap<String, Vec<f32>> = HashMap::new();
+        info!("Generating new embeddings for all chunks...");
+        for chunk in all_chunks.iter() {
+            let retry_strategy = ExponentialBackoff::from_millis(100).map(jitter).take(3);
+
+            let embedding = Retry::spawn(retry_strategy, || {
+                crate::utils::embedding::generate_embedding_with_params(
+                    openai_client,
+                    &chunk.chunk,
+                    new_model,
+                    new_dimensions,
+                )
+            })
+            .await?;
+
+            // Safety check: ensure the generated embedding has the correct dimension.
+            if embedding.len() != new_dimensions as usize {
+                let err_msg = format!(
+                    "CRITICAL: Generated embedding for chunk {} has incorrect dimension ({}). Expected {}. Aborting.",
+                    chunk.id, embedding.len(), new_dimensions
+                );
+                error!("{}", err_msg);
+                return Err(AppError::InternalError(err_msg));
+            }
+            new_embeddings.insert(chunk.id.clone(), embedding);
+        }
+        info!("Successfully generated all new embeddings.");
+
+        // Perform DB updates in a single transaction
+        info!("Applying schema and data changes in a transaction...");
+        let mut transaction_query = String::from("BEGIN TRANSACTION;");
+
+        // Add all update statements
+        for (id, embedding) in new_embeddings {
+            let embedding_str = format!(
+                "[{}]",
+                embedding
+                    .iter()
+                    .map(|f| f.to_string())
+                    .collect::<Vec<_>>()
+                    .join(",")
+            );
+            transaction_query.push_str(&format!(
+                "UPDATE type::thing('text_chunk', '{}') SET embedding = {}, updated_at = time::now();",
+                id, embedding_str
+            ));
+        }
+
+        // Re-create the index inside the same transaction
+        transaction_query.push_str("REMOVE INDEX idx_embedding_chunks ON TABLE text_chunk;");
+        transaction_query.push_str(&format!(
+            "DEFINE INDEX idx_embedding_chunks ON TABLE text_chunk FIELDS embedding HNSW DIMENSION {};",
+            new_dimensions
+        ));
+
+        transaction_query.push_str("COMMIT TRANSACTION;");
+
+        // Execute the entire atomic operation
+        db.query(transaction_query).await?;
+
+        info!("Re-embedding process for text chunks completed successfully.");
+        Ok(())
+    }
 }
 
 #[cfg(test)]

common/src/utils/config.rs

@@ -11,12 +11,19 @@ pub struct AppConfig {
     pub surrealdb_database: String,
     #[serde(default = "default_data_dir")]
     pub data_dir: String,
+    pub http_port: u16,
+    #[serde(default = "default_base_url")]
+    pub openai_base_url: String,
 }
 
 fn default_data_dir() -> String {
     "./data".to_string()
 }
 
+fn default_base_url() -> String {
+    "https://api.openai.com/v1".to_string()
+}
+
 pub fn get_config() -> Result<AppConfig, ConfigError> {
     let config = Config::builder()
         .add_source(File::with_name("config").required(false))

common/src/utils/embedding.rs

@@ -1,6 +1,10 @@
 use async_openai::types::CreateEmbeddingRequestArgs;
+use tracing::debug;
 
-use crate::error::AppError;
+use crate::{
+    error::AppError,
+    storage::{db::SurrealDbClient, types::system_settings::SystemSettings},
+};
 /// Generates an embedding vector for the given input text using OpenAI's embedding model.
 ///
 /// This function takes a text input and converts it into a numerical vector representation (embedding)
@@ -27,9 +31,13 @@ use crate::error::AppError;
 pub async fn generate_embedding(
     client: &async_openai::Client<async_openai::config::OpenAIConfig>,
     input: &str,
+    db: &SurrealDbClient,
 ) -> Result<Vec<f32>, AppError> {
+    let model = SystemSettings::get_current(db).await?;
+
     let request = CreateEmbeddingRequestArgs::default()
-        .model("text-embedding-3-small")
+        .model(model.embedding_model)
+        .dimensions(model.embedding_dimensions)
         .input([input])
         .build()?;
 
@@ -46,3 +54,36 @@ pub async fn generate_embedding(
 
     Ok(embedding)
 }
+
+/// Generates an embedding vector using a specific model and dimension.
+///
+/// This is used for the re-embedding process where the model and dimensions
+/// are known ahead of time and shouldn't be repeatedly fetched from settings.
+pub async fn generate_embedding_with_params(
+    client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+    input: &str,
+    model: &str,
+    dimensions: u32,
+) -> Result<Vec<f32>, AppError> {
+    let request = CreateEmbeddingRequestArgs::default()
+        .model(model)
+        .input([input])
+        .dimensions(dimensions as u32)
+        .build()?;
+
+    let response = client.embeddings().create(request).await?;
+
+    let embedding = response
+        .data
+        .first()
+        .ok_or_else(|| AppError::LLMParsing("No embedding data received from API".into()))?
+        .embedding
+        .clone();
+
+    debug!(
+        "Embedding was created with {:?} dimensions",
+        embedding.len()
+    );
+
+    Ok(embedding)
+}

composite-retrieval/src/vector.rs

@@ -1,6 +1,4 @@
-use surrealdb::{engine::any::Any, Surreal};
-
-use common::{error::AppError, utils::embedding::generate_embedding};
+use common::{error::AppError, storage::db::SurrealDbClient, utils::embedding::generate_embedding};
 
 /// Compares vectors and retrieves a number of items from the specified table.
 ///
@@ -26,7 +24,7 @@ use common::{error::AppError, utils::embedding::generate_embedding};
 pub async fn find_items_by_vector_similarity<T>(
     take: u8,
     input_text: &str,
-    db_client: &Surreal<Any>,
+    db_client: &SurrealDbClient,
     table: &str,
     openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
     user_id: &str,
@@ -35,7 +33,7 @@ where
     T: for<'de> serde::Deserialize<'de>,
 {
     // Generate embeddings
-    let input_embedding = generate_embedding(openai_client, input_text).await?;
+    let input_embedding = generate_embedding(openai_client, input_text, db_client).await?;
 
     // Construct the query
     let closest_query = format!("SELECT *, vector::distance::knn() AS distance FROM {} WHERE user_id = '{}' AND embedding <|{},40|> {:?} ORDER BY distance", table, user_id, take, input_embedding);

dist-workspace.toml

@@ -17,3 +17,4 @@ allow-dirty = ["ci"]
 [dist.github-custom-runners]
 x86_64-unknown-linux-gnu = "ubuntu-22.04"
 x86_64-unknown-linux-musl = "ubuntu-22.04"
+x86_64-pc-windows-msvc = "windows-latest"

docker-compose.yml

@@ -14,7 +14,8 @@ services:
       SURREALDB_NAMESPACE: "test"
       OPENAI_API_KEY: "sk-key"
       DATA_DIR: "./data"
-      # RUST_LOG: "info"
+      HTTP_PORT: 3000
+      RUST_LOG: "info"
     depends_on:
       - surrealdb 
     networks:

html-router/src/routes/admin/handlers.rs

@@ -1,13 +1,23 @@
+use async_openai::types::ListModelResponse;
 use axum::{extract::State, response::IntoResponse, Form};
 use serde::{Deserialize, Serialize};
 
-use common::storage::types::{
-    analytics::Analytics,
-    conversation::Conversation,
-    system_prompts::{DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT, DEFAULT_QUERY_SYSTEM_PROMPT},
-    system_settings::SystemSettings,
-    user::User,
+use common::{
+    error::AppError,
+    storage::types::{
+        analytics::Analytics,
+        conversation::Conversation,
+        knowledge_entity::KnowledgeEntity,
+        system_prompts::{
+            DEFAULT_IMAGE_PROCESSING_PROMPT, DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT,
+            DEFAULT_QUERY_SYSTEM_PROMPT,
+        },
+        system_settings::SystemSettings,
+        text_chunk::TextChunk,
+        user::User,
+    },
 };
+use tracing::{error, info};
 
 use crate::{
     html_state::HtmlState,
@@ -24,27 +34,41 @@ pub struct AdminPanelData {
     analytics: Analytics,
     users: i64,
     default_query_prompt: String,
+    default_image_prompt: String,
     conversation_archive: Vec<Conversation>,
+    available_models: ListModelResponse,
 }
 
 pub async fn show_admin_panel(
     State(state): State<HtmlState>,
     RequireUser(user): RequireUser,
 ) -> Result<impl IntoResponse, HtmlError> {
-    let settings = SystemSettings::get_current(&state.db).await?;
-    let analytics = Analytics::get_current(&state.db).await?;
-    let users_count = Analytics::get_users_amount(&state.db).await?;
-    let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
+    let (
+        settings_res,
+        analytics_res,
+        user_count_res,
+        conversation_archive_res,
+        available_models_res,
+    ) = tokio::join!(
+        SystemSettings::get_current(&state.db),
+        Analytics::get_current(&state.db),
+        Analytics::get_users_amount(&state.db),
+        User::get_user_conversations(&user.id, &state.db),
+        async { state.openai_client.models().list().await }
+    );
 
     Ok(TemplateResponse::new_template(
         "admin/base.html",
         AdminPanelData {
             user,
-            settings,
-            analytics,
-            users: users_count,
+            settings: settings_res?,
+            analytics: analytics_res?,
+            available_models: available_models_res
+                .map_err(|e| AppError::InternalError(e.to_string()))?,
+            users: user_count_res?,
             default_query_prompt: DEFAULT_QUERY_SYSTEM_PROMPT.to_string(),
-            conversation_archive,
+            default_image_prompt: DEFAULT_IMAGE_PROCESSING_PROMPT.to_string(),
+            conversation_archive: conversation_archive_res?,
         },
     ))
 }
@@ -103,11 +127,15 @@ pub async fn toggle_registration_status(
 pub struct ModelSettingsInput {
     query_model: String,
     processing_model: String,
+    image_processing_model: String,
+    embedding_model: String,
+    embedding_dimensions: Option<u32>,
 }
 
 #[derive(Serialize)]
 pub struct ModelSettingsData {
     settings: SystemSettings,
+    available_models: ListModelResponse,
 }
 
 pub async fn update_model_settings(
@@ -122,19 +150,76 @@ pub async fn update_model_settings(
 
     let current_settings = SystemSettings::get_current(&state.db).await?;
 
+    // Determine if re-embedding is required
+    let reembedding_needed = input
+        .embedding_dimensions
+        .is_some_and(|new_dims| new_dims != current_settings.embedding_dimensions);
+
     let new_settings = SystemSettings {
         query_model: input.query_model,
         processing_model: input.processing_model,
-        ..current_settings
+        image_processing_model: input.image_processing_model,
+        embedding_model: input.embedding_model,
+        // Use new dimensions if provided, otherwise retain the current ones.
+        embedding_dimensions: input
+            .embedding_dimensions
+            .unwrap_or(current_settings.embedding_dimensions),
+        ..current_settings.clone()
     };
 
     SystemSettings::update(&state.db, new_settings.clone()).await?;
 
+    if reembedding_needed {
+        info!("Embedding dimensions changed. Spawning background re-embedding task...");
+
+        let db_for_task = state.db.clone();
+        let openai_for_task = state.openai_client.clone();
+        let new_model_for_task = new_settings.embedding_model.clone();
+        let new_dims_for_task = new_settings.embedding_dimensions;
+
+        tokio::spawn(async move {
+            // First, update all text chunks
+            if let Err(e) = TextChunk::update_all_embeddings(
+                &db_for_task,
+                &openai_for_task,
+                &new_model_for_task,
+                new_dims_for_task,
+            )
+            .await
+            {
+                error!("Background re-embedding task failed for TextChunks: {}", e);
+            }
+
+            // Second, update all knowledge entities
+            if let Err(e) = KnowledgeEntity::update_all_embeddings(
+                &db_for_task,
+                &openai_for_task,
+                &new_model_for_task,
+                new_dims_for_task,
+            )
+            .await
+            {
+                error!(
+                    "Background re-embedding task failed for KnowledgeEntities: {}",
+                    e
+                );
+            }
+        });
+    }
+
+    let available_models = state
+        .openai_client
+        .models()
+        .list()
+        .await
+        .map_err(|_e| AppError::InternalError("Failed to get models".to_string()))?;
+
     Ok(TemplateResponse::new_partial(
         "admin/base.html",
         "model_settings_form",
         ModelSettingsData {
             settings: new_settings,
+            available_models,
         },
     ))
 }
@@ -261,3 +346,62 @@ pub async fn patch_ingestion_prompt(
         },
     ))
 }
+
+#[derive(Serialize)]
+pub struct ImagePromptEditData {
+    settings: SystemSettings,
+    default_image_prompt: String,
+}
+
+pub async fn show_edit_image_prompt(
+    State(state): State<HtmlState>,
+    RequireUser(user): RequireUser,
+) -> Result<impl IntoResponse, HtmlError> {
+    // Early return if the user is not admin
+    if !user.admin {
+        return Ok(TemplateResponse::redirect("/"));
+    };
+
+    let settings = SystemSettings::get_current(&state.db).await?;
+
+    Ok(TemplateResponse::new_template(
+        "admin/edit_image_prompt_modal.html",
+        ImagePromptEditData {
+            settings,
+            default_image_prompt: DEFAULT_IMAGE_PROCESSING_PROMPT.to_string(),
+        },
+    ))
+}
+
+#[derive(Deserialize)]
+pub struct ImagePromptUpdateInput {
+    image_processing_prompt: String,
+}
+
+pub async fn patch_image_prompt(
+    State(state): State<HtmlState>,
+    RequireUser(user): RequireUser,
+    Form(input): Form<ImagePromptUpdateInput>,
+) -> Result<impl IntoResponse, HtmlError> {
+    // Early return if the user is not admin
+    if !user.admin {
+        return Ok(TemplateResponse::redirect("/"));
+    };
+
+    let current_settings = SystemSettings::get_current(&state.db).await?;
+
+    let new_settings = SystemSettings {
+        image_processing_prompt: input.image_processing_prompt,
+        ..current_settings.clone()
+    };
+
+    SystemSettings::update(&state.db, new_settings.clone()).await?;
+
+    Ok(TemplateResponse::new_partial(
+        "admin/base.html",
+        "system_prompt_section",
+        SystemPromptSectionData {
+            settings: new_settings,
+        },
+    ))
+}

html-router/src/routes/admin/mod.rs

@@ -5,8 +5,9 @@ use axum::{
     Router,
 };
 use handlers::{
-    patch_ingestion_prompt, patch_query_prompt, show_admin_panel, show_edit_ingestion_prompt,
-    show_edit_system_prompt, toggle_registration_status, update_model_settings,
+    patch_image_prompt, patch_ingestion_prompt, patch_query_prompt, show_admin_panel,
+    show_edit_image_prompt, show_edit_ingestion_prompt, show_edit_system_prompt,
+    toggle_registration_status, update_model_settings,
 };
 
 use crate::html_state::HtmlState;
@@ -24,4 +25,6 @@ where
         .route("/update-query-prompt", patch(patch_query_prompt))
         .route("/edit-ingestion-prompt", get(show_edit_ingestion_prompt))
         .route("/update-ingestion-prompt", patch(patch_ingestion_prompt))
+        .route("/edit-image-prompt", get(show_edit_image_prompt))
+        .route("/update-image-prompt", patch(patch_image_prompt))
 }

html-router/src/routes/content/handlers.rs

@@ -7,7 +7,7 @@ use axum_htmx::{HxBoosted, HxRequest};
 use serde::{Deserialize, Serialize};
 
 use common::storage::types::{
-    conversation::Conversation, file_info::FileInfo, text_content::TextContent, user::User,
+    conversation::Conversation, file_info::FileInfo, text_content::TextContent, user::User, knowledge_entity::KnowledgeEntity, text_chunk::TextChunk,
 };
 
 use crate::{
@@ -138,6 +138,10 @@ pub async fn delete_text_content(
         FileInfo::delete_by_id(&file_info.id, &state.db).await?;
     }
 
+    // Delete related knowledge entities and text chunks
+    KnowledgeEntity::delete_by_source_id(&id, &state.db).await?;
+    TextChunk::delete_by_source_id(&id, &state.db).await?;
+
     // Delete the text content
     state.db.delete_item::<TextContent>(&id).await?;
 
@@ -190,7 +194,7 @@ pub async fn show_recent_content(
     }
 
     Ok(TemplateResponse::new_template(
-        "/dashboard/recent_content.html",
+        "dashboard/recent_content.html",
         RecentTextContentData {
             user,
             text_contents,

html-router/src/routes/index/handlers.rs

@@ -138,7 +138,7 @@ pub async fn delete_job(
     let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
 
     Ok(TemplateResponse::new_partial(
-        "index/signed_in/active_jobs.html",
+        "dashboard/active_jobs.html",
         "active_jobs_section",
         ActiveJobsData {
             user: user.clone(),

html-router/src/routes/ingestion/handlers.rs

@@ -186,9 +186,9 @@ pub async fn get_task_updates_stream(
                                     format!("In progress, attempt {}", attempts)
                                 }
                                 IngestionTaskStatus::Completed => "Completed".to_string(),
-                                IngestionTaskStatus::Error(ref err_msg) => {
+                                IngestionTaskStatus::Error { message } => {
                                     // Providing a user-friendly error message from the status
-                                    format!("Error: {}", err_msg)
+                                    format!("Error: {}", message)
                                 }
                                 IngestionTaskStatus::Cancelled => "Cancelled".to_string(),
                             };
@@ -197,9 +197,9 @@ pub async fn get_task_updates_stream(
 
                             // Check for terminal states to close the stream
                             match updated_task.status {
-                                IngestionTaskStatus::Completed |
-                                IngestionTaskStatus::Error(_) |
-                                IngestionTaskStatus::Cancelled => {
+                                IngestionTaskStatus::Completed
+                                | IngestionTaskStatus::Error { .. }
+                                | IngestionTaskStatus::Cancelled => {
                                     // Send a specific event that HTMX uses to close the connection
                                     // Send a event to reload the recent content
                                     // Send a event to remove the loading indicatior

html-router/templates/admin/base.html

@@ -6,7 +6,7 @@
 <main class="container flex-grow flex flex-col mx-auto mt-4 space-y-6">
   <h1 class="text-2xl font-bold mb-2">Admin Dashboard</h1>
 
-  <div class="stats stats-vertical lg:stats-horizontal shadow">
+  <div class="stats stats-vertical md:stats-horizontal shadow">
     <div class="stat">
       <div class="stat-title font-bold">Page loads</div>
       <div class="stat-value text-secondary">{{analytics.page_loads}}</div>
@@ -27,7 +27,7 @@
   </div>
 
   <!-- Settings in Fieldset -->
-  <div class="grid grid-cols-1 sm:grid-cols-2 gap-6">
+  <div class="grid grid-cols-1 xl:grid-cols-2 gap-6">
     {% block system_prompt_section %}
     <div id="system_prompt_section">
       <fieldset class="fieldset p-4 shadow rounded-box">
@@ -41,6 +41,10 @@
             hx-swap="innerHTML">
             Edit Ingestion Prompt
           </button>
+          <button type="button" class="btn btn-primary btn-sm" hx-get="/edit-image-prompt" hx-target="#modal"
+            hx-swap="innerHTML">
+            Edit Image Prompt
+          </button>
         </div>
       </fieldset>
     </div>
@@ -50,36 +54,121 @@
       <legend class="fieldset-legend">AI Models</legend>
       {% block model_settings_form %}
       <form hx-patch="/update-model-settings" hx-swap="outerHTML">
+        <!-- Query Model -->
         <div class="form-control mb-4">
           <label class="label">
             <span class="label-text">Query Model</span>
           </label>
           <select name="query_model" class="select select-bordered w-full">
-            <option value="gpt-4o-mini" {% if settings.query_model=="gpt-4o-mini" %}selected{% endif %}>GPT-4o Mini
-            </option>
-            <option value="gpt-4.1" {% if settings.query_model=="gpt-4.1" %}selected{% endif %}>GPT-4.1</option>
-            <option value="gpt-4.1-mini" {% if settings.query_model=="gpt-4.1-mini" %}selected{% endif %}>GPT-4.1-mini
+            {% for model in available_models.data %}
+            <option value="{{model.id}}" {% if settings.query_model==model.id %} selected {% endif %}>{{model.id}}
             </option>
+            {% endfor %}
           </select>
-          <p class="text-xs text-gray-500 mt-1">Model used for answering user queries</p>
+
+          <p class="text-xs text-gray-500 mt-1">
+            Current used:
+            <span class="font-mono">{{settings.query_model}}</span>
+          </p>
         </div>
 
-        <div class="form-control my-4">
+        <!-- Processing Model -->
+        <div class="form-control mb-4">
           <label class="label">
             <span class="label-text">Processing Model</span>
           </label>
           <select name="processing_model" class="select select-bordered w-full">
-            <option value="gpt-4o-mini" {% if settings.query_model=="gpt-4o-mini" %}selected{% endif %}>GPT-4o Mini
-            </option>
-            <option value="gpt-4.1" {% if settings.query_model=="gpt-4.1" %}selected{% endif %}>GPT-4.1</option>
-            <option value="gpt-4.1-mini" {% if settings.query_model=="gpt-4.1-mini" %}selected{% endif %}>GPT-4.1-mini
+            {% for model in available_models.data %}
+            <option value="{{model.id}}" {% if settings.processing_model==model.id %} selected {% endif %}>{{model.id}}
             </option>
+            {% endfor %}
           </select>
-          <p class="text-xs text-gray-500 mt-1">Model used for content processing and ingestion</p>
+          <p class="text-xs text-gray-500 mt-1">
+            Current used:
+            <span class="font-mono">{{settings.processing_model}}</span>
+          </p>
         </div>
 
-        <button type="submit" class="btn btn-primary btn-sm">Save Model Settings</button>
+        <!-- Image Processing Model -->
+        <div class="form-control mb-4">
+          <label class="label">
+            <span class="label-text">Image Processing Model</span>
+          </label>
+          <select name="image_processing_model" class="select select-bordered w-full">
+            {% for model in available_models.data %}
+            <option value="{{model.id}}" {% if settings.image_processing_model==model.id %} selected {% endif %}>
+              {{model.id}}
+            </option>
+            {% endfor %}
+          </select>
+          <p class="text-xs text-gray-500 mt-1">
+            Current used:
+            <span class="font-mono">{{settings.image_processing_model}}</span>
+          </p>
+        </div>
+
+        <!-- Embedding Model -->
+        <div class="form-control mb-4">
+          <label class="label">
+            <span class="label-text">Embedding Model</span>
+          </label>
+          <select name="embedding_model" class="select select-bordered w-full">
+            {% for model in available_models.data %}
+            <option value="{{model.id}}" {% if settings.embedding_model==model.id %} selected {% endif %}>{{model.id}}
+            </option>
+            {% endfor %}
+          </select>
+          <p class="text-xs text-gray-500 mt-1">
+            Current used:
+            <span class="font-mono">{{settings.embedding_model}} ({{settings.embedding_dimensions}} dims)</span>
+          </p>
+        </div>
+
+        <!-- Embedding Dimensions (Always Visible) -->
+        <div class="form-control mb-4">
+          <label class="label" for="embedding_dimensions">
+            <span class="label-text">Embedding Dimensions</span>
+          </label>
+          <input type="number" id="embedding_dimensions" name="embedding_dimensions" class="input input-bordered w-full"
+            value="{{ settings.embedding_dimensions }}" required />
+        </div>
+
+        <!-- Conditional Alert -->
+        <div id="embedding-change-alert" role="alert" class="alert alert-warning mt-2 hidden">
+          <svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none"
+            viewBox="0 0 24 24">
+            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
+              d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
+          </svg>
+          <span><strong>Warning:</strong> Changing dimensions will require re-creating all embeddings. Make sure you
+            look up what dimensions the model uses or use a model that allows specifying embedding dimensions</span>
+        </div>
+
+        <button type="submit" class="btn btn-primary btn-sm mt-4">Save Model Settings</button>
       </form>
+
+      <script>
+        // Use a self-executing function to avoid polluting the global scope
+        // and to ensure it runs correctly after an HTMX swap.
+        (() => {
+          const dimensionInput = document.getElementById('embedding_dimensions');
+          const alertElement = document.getElementById('embedding-change-alert');
+          // The initial value is read directly from the template each time this script runs.
+          const initialDimensions = '{{ settings.embedding_dimensions }}';
+
+          if (dimensionInput && alertElement) {
+            // Use the 'input' event for immediate feedback as the user types.
+            dimensionInput.addEventListener('input', (event) => {
+              // Show alert if the current value is not the initial value. Hide it otherwise.
+              if (event.target.value !== initialDimensions) {
+                alertElement.classList.remove('hidden');
+              } else {
+                alertElement.classList.add('hidden');
+              }
+            });
+          }
+        })();
+      </script>
       {% endblock %}
     </fieldset>
 

html-router/templates/admin/edit_image_prompt_modal.html

@@ -0,0 +1,38 @@
+{% extends "modal_base.html" %}
+
+{% block form_attributes %}
+hx-patch="/update-image-prompt"
+hx-target="#system_prompt_section"
+hx-swap="outerHTML"
+{% endblock %}
+
+{% block modal_content %}
+<h3 class="text-lg font-bold mb-4">Edit Image Processing Prompt</h3>
+
+<div class="form-control">
+  <textarea name="image_processing_prompt" class="textarea textarea-bordered h-96 w-full font-mono text-sm">{{
+    settings.image_processing_prompt }}</textarea>
+  <p class="text-xs text-gray-500 mt-1">System prompt used for processing images</p>
+</div>
+{% endblock %}
+
+{% block primary_actions %}
+<button type="button" class="btn btn-outline mr-2" id="reset_prompt_button">
+  Reset to Default
+</button>
+
+<textarea id="default_prompt_content" style="display:none;">{{ default_image_prompt }}</textarea>
+<script>
+  document.getElementById('reset_prompt_button').addEventListener('click', function () {
+    const defaultContent = document.getElementById('default_prompt_content').value;
+    document.querySelector('textarea[name=image_processing_prompt]').value = defaultContent;
+  });
+</script>
+
+<button type="submit" class="btn btn-primary">
+  <span class="htmx-indicator hidden">
+    <span class="loading loading-spinner loading-xs mr-2"></span>
+  </span>
+  Save Changes
+</button>
+{% endblock %}

html-router/templates/content/content_list.html

@@ -7,18 +7,21 @@
       <img src="/file/{{text_content.url_info.image_id}}" alt="website screenshot" />
     </figure>
     {% endif %}
-    <div class="card-body">
-      <div class="flex justify-between space-x-2">
-        <h2 class="card-title truncate">
-          {% if text_content.url_info %}
-          {{text_content.url_info.title}}
-          {% elif text_content.file_info %}
-          {{text_content.file_info.file_name}}
-          {% else %}
-          {{text_content.text}}
-          {% endif %}
-        </h2>
-      </div>
+    {% if text_content.file_info.mime_type == "image/png" or text_content.file_info.mime_type == "image/jpeg" %}
+    <figure>
+      <img src="/file/{{text_content.file_info.id}}" alt="{{text_content.file_info.file_name}}" />
+    </figure>
+    {% endif %}
+    <div class="card-body max-w-[95vw]">
+      <h2 class="card-title truncate">
+        {% if text_content.url_info %}
+        {{text_content.url_info.title}}
+        {% elif text_content.file_info %}
+        {{text_content.file_info.file_name}}
+        {% else %}
+        {{text_content.text}}
+        {% endif %}
+      </h2>
       <div class="flex items-center justify-between">
         <p class="text-xs opacity-60">
           {{ text_content.created_at | datetimeformat(format="short", tz=user.timezone) }}

html-router/templates/content/read_content_modal.html

@@ -6,6 +6,11 @@
 {% if text_content.url_info.image_id %}
 <img class="rounded-t-md overflow-clip" src="/file/{{text_content.url_info.image_id}}" alt="Screenshot of the site" />
 {% endif %}
+{% if text_content.file_info.mime_type == "image/png" or text_content.file_info.mime_type == "image/jpeg" %}
+<figure>
+  <img src="/file/{{text_content.file_info.id}}" alt="{{text_content.file_info.file_name}}" />
+</figure>
+{% endif %}
 <div id="reader-{{text_content.id}}" class="markdown-content prose" data-content="{{text_content.text | escape }}">
   {{text_content.text | escape }}
 </div>

html-router/templates/dashboard/active_jobs.html

@@ -19,10 +19,12 @@
     </div>
     <div>
       <div class="[&:before]:content-['Status:_'] [&:before]:opacity-60">
-        {% if item.status.InProgress %}
-        In Progress, attempt {{item.status.InProgress.attempts}}
+        {% if item.status.name == "InProgress" %}
+        In Progress, attempt {{item.status.attempts}}
+        {% elif item.status.name == "Error" %}
+        Error: {{item.status.message}}
         {% else %}
-        {{item.status}}
+        {{item.status.name}}
         {% endif %}
       </div>
       <div class="text-xs font-semibold opacity-60">

ingestion-pipeline/Cargo.toml

@@ -22,9 +22,9 @@ text-splitter = { workspace = true }
 url = { workspace = true }
 uuid = { workspace = true }
 headless_chrome = { workspace = true }
+base64 = { workspace = true }
 
 common = { path = "../common" }
 composite-retrieval = { path = "../composite-retrieval" }
-
 [features]
 docker = []

ingestion-pipeline/src/lib.rs

@@ -44,7 +44,7 @@ pub async fn run_worker_loop(
                         Action::Update => {
                             match notification.data.status {
                                 IngestionTaskStatus::Completed
-                                | IngestionTaskStatus::Error(_)
+                                | IngestionTaskStatus::Error { .. }
                                 | IngestionTaskStatus::Cancelled => {
                                     info!(
                                         "Skipping already completed/error/cancelled task: {}",
@@ -58,7 +58,7 @@ pub async fn run_worker_loop(
                                         db.get_item::<IngestionTask>(&notification.data.id).await
                                     {
                                         match current_task.status {
-                                              IngestionTaskStatus::Error(_)
+                                            IngestionTaskStatus::Error { .. }
                                                   if attempts
                                                       < common::storage::types::ingestion_task::MAX_ATTEMPTS =>
                                               {

ingestion-pipeline/src/pipeline.rs

@@ -59,7 +59,8 @@ impl IngestionPipeline {
         )
         .await?;
 
-        let text_content = to_text_content(task.content, &self.db, &self.config).await?;
+        let text_content =
+            to_text_content(task.content, &self.db, &self.config, &self.openai_client).await?;
 
         match self.process(&text_content).await {
             Ok(_) => {
@@ -71,7 +72,9 @@ impl IngestionPipeline {
                 if current_attempts >= MAX_ATTEMPTS {
                     IngestionTask::update_status(
                         &task.id,
-                        IngestionTaskStatus::Error(format!("Max attempts reached: {}", e)),
+                        IngestionTaskStatus::Error {
+                            message: format!("Max attempts reached: {}", e),
+                        },
                         &self.db,
                     )
                     .await?;
@@ -95,7 +98,7 @@ impl IngestionPipeline {
 
         // Convert analysis to application objects
         let (entities, relationships) = analysis
-            .to_database_entities(&content.id, &content.user_id, &self.openai_client)
+            .to_database_entities(&content.id, &content.user_id, &self.openai_client, &self.db)
             .await?;
 
         // Store everything
@@ -155,7 +158,7 @@ impl IngestionPipeline {
 
         // Could potentially process chunks in parallel with a bounded concurrent limit
         for chunk in chunks {
-            let embedding = generate_embedding(&self.openai_client, chunk).await?;
+            let embedding = generate_embedding(&self.openai_client, chunk, &self.db).await?;
             let text_chunk = TextChunk::new(
                 content.id.to_string(),
                 chunk.to_string(),

ingestion-pipeline/src/types/llm_enrichment_result.rs

@@ -6,9 +6,12 @@ use tokio::task;
 
 use common::{
     error::AppError,
-    storage::types::{
-        knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
-        knowledge_relationship::KnowledgeRelationship,
+    storage::{
+        db::SurrealDbClient,
+        types::{
+            knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
+            knowledge_relationship::KnowledgeRelationship,
+        },
     },
     utils::embedding::generate_embedding,
 };
@@ -56,13 +59,20 @@ impl LLMEnrichmentResult {
         source_id: &str,
         user_id: &str,
         openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+        db_client: &SurrealDbClient,
     ) -> Result<(Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>), AppError> {
         // Create mapper and pre-assign IDs
         let mapper = Arc::new(Mutex::new(self.create_mapper()?));
 
         // Process entities
         let entities = self
-            .process_entities(source_id, user_id, Arc::clone(&mapper), openai_client)
+            .process_entities(
+                source_id,
+                user_id,
+                Arc::clone(&mapper),
+                openai_client,
+                db_client,
+            )
             .await?;
 
         // Process relationships
@@ -88,6 +98,7 @@ impl LLMEnrichmentResult {
         user_id: &str,
         mapper: Arc<Mutex<GraphMapper>>,
         openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+        db_client: &SurrealDbClient,
     ) -> Result<Vec<KnowledgeEntity>, AppError> {
         let futures: Vec<_> = self
             .knowledge_entities
@@ -98,10 +109,18 @@ impl LLMEnrichmentResult {
                 let source_id = source_id.to_string();
                 let user_id = user_id.to_string();
                 let entity = entity.clone();
+                let db_client = db_client.clone();
 
                 task::spawn(async move {
-                    create_single_entity(&entity, &source_id, &user_id, mapper, &openai_client)
-                        .await
+                    create_single_entity(
+                        &entity,
+                        &source_id,
+                        &user_id,
+                        mapper,
+                        &openai_client,
+                        &db_client.clone(),
+                    )
+                    .await
                 })
             })
             .collect();
@@ -120,14 +139,14 @@ impl LLMEnrichmentResult {
         user_id: &str,
         mapper: Arc<Mutex<GraphMapper>>,
     ) -> Result<Vec<KnowledgeRelationship>, AppError> {
-        let mut mapper_guard = mapper
+        let mapper_guard = mapper
             .lock()
             .map_err(|_| AppError::GraphMapper("Failed to lock mapper".into()))?;
         self.relationships
             .iter()
             .map(|rel| {
-                let source_db_id = mapper_guard.get_or_parse_id(&rel.source);
-                let target_db_id = mapper_guard.get_or_parse_id(&rel.target);
+                let source_db_id = mapper_guard.get_or_parse_id(&rel.source)?;
+                let target_db_id = mapper_guard.get_or_parse_id(&rel.target)?;
 
                 Ok(KnowledgeRelationship::new(
                     source_db_id.to_string(),
@@ -146,17 +165,13 @@ async fn create_single_entity(
     user_id: &str,
     mapper: Arc<Mutex<GraphMapper>>,
     openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+    db_client: &SurrealDbClient,
 ) -> Result<KnowledgeEntity, AppError> {
     let assigned_id = {
         let mapper = mapper
             .lock()
             .map_err(|_| AppError::GraphMapper("Failed to lock mapper".into()))?;
-        mapper
-            .get_id(&llm_entity.key)
-            .ok_or_else(|| {
-                AppError::GraphMapper(format!("ID not found for key: {}", llm_entity.key))
-            })?
-            .to_string()
+        mapper.get_id(&llm_entity.key)?.to_string()
     };
 
     let embedding_input = format!(
@@ -164,7 +179,7 @@ async fn create_single_entity(
         llm_entity.name, llm_entity.description, llm_entity.entity_type
     );
 
-    let embedding = generate_embedding(openai_client, &embedding_input).await?;
+    let embedding = generate_embedding(openai_client, &embedding_input, db_client).await?;
 
     let now = Utc::now();
     Ok(KnowledgeEntity {

ingestion-pipeline/src/types/mod.rs

@@ -22,10 +22,13 @@ use std::io::{Seek, SeekFrom};
 use tempfile::NamedTempFile;
 use tracing::{error, info};
 
+use crate::utils::image_parsing::extract_text_from_image;
+
 pub async fn to_text_content(
     ingestion_payload: IngestionPayload,
     db: &SurrealDbClient,
     config: &AppConfig,
+    openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
 ) -> Result<TextContent, AppError> {
     match ingestion_payload {
         IngestionPayload::Url {
@@ -67,7 +70,7 @@ pub async fn to_text_content(
             category,
             user_id,
         } => {
-            let text = extract_text_from_file(&file_info).await?;
+            let text = extract_text_from_file(&file_info, db, openai_client).await?;
             Ok(TextContent::new(
                 text,
                 Some(context),
@@ -195,7 +198,11 @@ async fn fetch_article_from_url(
 }
 
 /// Extracts text from a file based on its MIME type.
-async fn extract_text_from_file(file_info: &FileInfo) -> Result<String, AppError> {
+async fn extract_text_from_file(
+    file_info: &FileInfo,
+    db_client: &SurrealDbClient,
+    openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+) -> Result<String, AppError> {
     match file_info.mime_type.as_str() {
         "text/plain" => {
             // Read the file and return its content
@@ -212,8 +219,9 @@ async fn extract_text_from_file(file_info: &FileInfo) -> Result<String, AppError
             Err(AppError::NotFound(file_info.mime_type.clone()))
         }
         "image/png" | "image/jpeg" => {
-            // TODO: Implement OCR on image using a crate like `tesseract`
-            Err(AppError::NotFound(file_info.mime_type.clone()))
+            let content =
+                extract_text_from_image(&file_info.path, db_client, openai_client).await?;
+            Ok(content)
         }
         "application/octet-stream" => {
             let content = tokio::fs::read_to_string(&file_info.path).await?;

ingestion-pipeline/src/utils/image_parsing.rs

@@ -0,0 +1,57 @@
+use async_openai::types::{
+    ChatCompletionRequestMessageContentPartImageArgs,
+    ChatCompletionRequestMessageContentPartTextArgs, ChatCompletionRequestUserMessageArgs,
+    CreateChatCompletionRequestArgs, ImageDetail, ImageUrlArgs,
+};
+use base64::{engine::general_purpose::STANDARD, Engine as _};
+use common::{
+    error::AppError,
+    storage::{db::SurrealDbClient, types::system_settings::SystemSettings},
+};
+
+pub async fn extract_text_from_image(
+    path: &str,
+    db: &SurrealDbClient,
+    client: &async_openai::Client<async_openai::config::OpenAIConfig>,
+) -> Result<String, AppError> {
+    let system_settings = SystemSettings::get_current(db).await?;
+    let image_bytes = tokio::fs::read(&path).await?;
+
+    let base64_image = STANDARD.encode(&image_bytes);
+
+    let image_url = format!("data:image/png;base64,{}", base64_image);
+
+    let request = CreateChatCompletionRequestArgs::default()
+        .model(system_settings.image_processing_model)
+        .max_tokens(6400_u32)
+        .messages([ChatCompletionRequestUserMessageArgs::default()
+            .content(vec![
+                ChatCompletionRequestMessageContentPartTextArgs::default()
+                    .text(system_settings.image_processing_prompt)
+                    .build()?
+                    .into(),
+                ChatCompletionRequestMessageContentPartImageArgs::default()
+                    .image_url(
+                        ImageUrlArgs::default()
+                            .url(image_url)
+                            .detail(ImageDetail::High)
+                            .build()?,
+                    )
+                    .build()?
+                    .into(),
+            ])
+            .build()?
+            .into()])
+        .build()?;
+
+    let response = client.chat().create(request).await?;
+
+    let description = response
+        .choices
+        .get(0)
+        .and_then(|c| c.message.content.as_ref())
+        .cloned()
+        .unwrap_or_else(|| "No description found.".to_string());
+
+    Ok(description)
+}

ingestion-pipeline/src/utils/mod.rs

@@ -1,5 +1,7 @@
+pub mod image_parsing;
 pub mod llm_instructions;
 
+use common::error::AppError;
 use std::collections::HashMap;
 use uuid::Uuid;
 
@@ -21,24 +23,39 @@ impl GraphMapper {
             key_to_id: HashMap::new(),
         }
     }
-    /// Get ID, tries to parse UUID
-    pub fn get_or_parse_id(&mut self, key: &str) -> Uuid {
+    /// Tries to get an ID by first parsing the key as a UUID,
+    /// and if that fails, looking it up in the internal map.
+    pub fn get_or_parse_id(&self, key: &str) -> Result<Uuid, AppError> {
+        // First, try to parse the key as a UUID.
         if let Ok(parsed_uuid) = Uuid::parse_str(key) {
-            parsed_uuid
-        } else {
-            *self.key_to_id.get(key).unwrap()
+            return Ok(parsed_uuid);
         }
+
+        // If parsing fails, look it up in the map.
+        self.key_to_id
+            .get(key)
+            .map(|id| *id) // Dereference the &Uuid to get Uuid
+            // If `get` returned None, create and return an error.
+            .ok_or_else(|| {
+                AppError::GraphMapper(format!(
+                    "Key '{}' is not a valid UUID and was not found in the map.",
+                    key
+                ))
+            })
     }
 
-    /// Assigns a new UUID for a given key.
+    /// Assigns a new UUID for a given key. (No changes needed here)
     pub fn assign_id(&mut self, key: &str) -> Uuid {
         let id = Uuid::new_v4();
         self.key_to_id.insert(key.to_string(), id);
         id
     }
 
-    /// Retrieves the UUID for a given key.
-    pub fn get_id(&self, key: &str) -> Option<&Uuid> {
-        self.key_to_id.get(key)
+    /// Retrieves the UUID for a given key, returning a Result for consistency.
+    pub fn get_id(&self, key: &str) -> Result<Uuid, AppError> {
+        self.key_to_id
+            .get(key)
+            .map(|id| *id)
+            .ok_or_else(|| AppError::GraphMapper(format!("Key '{}' not found in map.", key)))
     }
 }

json-stream-parser/src/lib.rs

@@ -43,7 +43,6 @@ fn add_char_into_object(
     current_char: char,
 ) -> Result<(), String> {
     match (&*object, &*current_status, current_char) {
-        // String escape handling
         (&Value::String(_), &ObjectStatus::StringQuoteOpen(true), '"') => {
             if let Value::String(str) = object {
                 str.push('"');
@@ -54,7 +53,6 @@ fn add_char_into_object(
             *current_status = ObjectStatus::StringQuoteClose;
         }
         (&Value::String(_), &ObjectStatus::StringQuoteOpen(true), c) => {
-            // Handle other escaped chars
             if let Value::String(str) = object {
                 str.push('\\');
                 str.push(c);
@@ -70,7 +68,6 @@ fn add_char_into_object(
             }
         }
 
-        // Key escape handling
         (&Value::Object(_), &ObjectStatus::KeyQuoteOpen { escaped: true, .. }, '"') => {
             if let ObjectStatus::KeyQuoteOpen {
                 ref mut key_so_far, ..
@@ -124,7 +121,6 @@ fn add_char_into_object(
             }
         }
 
-        // Value string escape handling
         (&Value::Object(_), &ObjectStatus::ValueQuoteOpen { escaped: true, .. }, '"') => {
             if let ObjectStatus::ValueQuoteOpen { ref key, .. } = current_status {
                 let key_str = key.iter().collect::<String>();
@@ -178,7 +174,6 @@ fn add_char_into_object(
             }
         }
 
-        // All other cases from the original implementation
         (&Value::Null, &ObjectStatus::Ready, '"') => {
             *object = json!("");
             *current_status = ObjectStatus::StringQuoteOpen(false);
@@ -188,7 +183,6 @@ fn add_char_into_object(
             *current_status = ObjectStatus::StartProperty;
         }
 
-        // ------ true ------
         (&Value::Null, &ObjectStatus::Ready, 't') => {
             *object = json!(true);
             *current_status = ObjectStatus::Scalar {
@@ -219,7 +213,6 @@ fn add_char_into_object(
             *current_status = ObjectStatus::Closed;
         }
 
-        // ------ false ------
         (&Value::Null, &ObjectStatus::Ready, 'f') => {
             *object = json!(false);
             *current_status = ObjectStatus::Scalar {
@@ -260,7 +253,6 @@ fn add_char_into_object(
             *current_status = ObjectStatus::Closed;
         }
 
-        // ------ null ------
         (&Value::Null, &ObjectStatus::Ready, 'n') => {
             *object = json!(null);
             *current_status = ObjectStatus::Scalar {
@@ -285,13 +277,11 @@ fn add_char_into_object(
                 if *value_so_far == vec!['n', 'u'] {
                     value_so_far.push('l');
                 } else if *value_so_far == vec!['n', 'u', 'l'] {
-                    // This is for the second 'l' in "null"
                     *current_status = ObjectStatus::Closed;
                 }
             }
         }
 
-        // ------ number ------
         (&Value::Null, &ObjectStatus::Ready, c @ '0'..='9') => {
             *object = Value::Number(c.to_digit(10).unwrap().into());
             *current_status = ObjectStatus::ScalarNumber {
@@ -310,7 +300,6 @@ fn add_char_into_object(
             } = current_status
             {
                 value_so_far.push(c);
-                // Parse based on whether it's a float or int
                 if let Value::Number(ref mut num) = object {
                     if value_so_far.contains(&'.') {
                         let parsed_number = value_so_far
@@ -341,7 +330,6 @@ fn add_char_into_object(
             }
         }
 
-        // Object handling
         (&Value::Object(_), &ObjectStatus::StartProperty, '"') => {
             *current_status = ObjectStatus::KeyQuoteOpen {
                 key_so_far: vec![],
@@ -367,7 +355,6 @@ fn add_char_into_object(
             }
         }
 
-        // ------ Add Scalar Value ------
         (&Value::Object(_), &ObjectStatus::Colon { .. }, char) => {
             if let ObjectStatus::Colon { ref key } = current_status {
                 *current_status = ObjectStatus::ValueScalar {
@@ -424,7 +411,6 @@ fn add_char_into_object(
             }
         }
 
-        // ------ Finished taking value ------
         (&Value::Object(_), &ObjectStatus::ValueQuoteClose, ',') => {
             *current_status = ObjectStatus::StartProperty;
         }
@@ -432,7 +418,6 @@ fn add_char_into_object(
             *current_status = ObjectStatus::Closed;
         }
 
-        // ------ white spaces ------
         (_, _, ' ' | '\n') => {}
 
         (val, st, c) => {
@@ -445,7 +430,6 @@ fn add_char_into_object(
     Ok(())
 }
 
-// The rest of the code remains the same
 #[cfg(debug_assertions)]
 pub fn parse_stream(json_string: &str) -> Result<Value, String> {
     let mut out: Value = Value::Null;

main/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "main"
-version = "0.1.2"
+version = "0.1.4"
 edition = "2021"
 repository = "https://github.com/perstarkse/minne"
 license = "AGPL-3.0-or-later"

main/src/main.rs

@@ -38,7 +38,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let session_store = Arc::new(db.create_session_store().await?);
     let openai_client = Arc::new(async_openai::Client::with_config(
-        async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
+        async_openai::config::OpenAIConfig::new()
+            .with_api_key(&config.openai_api_key)
+            .with_api_base(&config.openai_base_url),
     ));
 
     let html_state =
@@ -58,8 +60,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             html_state,
         });
 
-    info!("Starting server listening on 0.0.0.0:3000");
-    let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
+    info!("Starting server listening on 0.0.0.0:{}", config.http_port);
+    let serve_address = format!("0.0.0.0:{}", config.http_port);
+    let listener = tokio::net::TcpListener::bind(serve_address).await?;
 
     // Start the server in a separate OS thread with its own runtime
     let server_handle = std::thread::spawn(move || {
@@ -93,7 +96,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         );
 
         // Initialize worker components
-        let openai_client = Arc::new(async_openai::Client::new());
+        let openai_client = Arc::new(async_openai::Client::with_config(
+            async_openai::config::OpenAIConfig::new()
+                .with_api_key(&config.openai_api_key)
+                .with_api_base(&config.openai_base_url),
+        ));
         let ingestion_pipeline = Arc::new(
             IngestionPipeline::new(worker_db.clone(), openai_client.clone(), config.clone())
                 .await

main/src/server.rs

@@ -36,7 +36,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let session_store = Arc::new(db.create_session_store().await?);
     let openai_client = Arc::new(async_openai::Client::with_config(
-        async_openai::config::OpenAIConfig::new().with_api_key(&config.openai_api_key),
+        async_openai::config::OpenAIConfig::new()
+            .with_api_key(&config.openai_api_key)
+            .with_api_base(&config.openai_base_url),
     ));
 
     let html_state =
@@ -56,8 +58,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             html_state,
         });
 
-    info!("Listening on 0.0.0.0:3000");
-    let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
+    info!("Starting server listening on 0.0.0.0:{}", config.http_port);
+    let serve_address = format!("0.0.0.0:{}", config.http_port);
+    let listener = tokio::net::TcpListener::bind(serve_address).await?;
     axum::serve(listener, app).await?;
 
     Ok(())

main/src/worker.rs

@@ -26,7 +26,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .await?,
     );
 
-    let openai_client = Arc::new(async_openai::Client::new());
+    let openai_client = Arc::new(async_openai::Client::with_config(
+        async_openai::config::OpenAIConfig::new()
+            .with_api_key(&config.openai_api_key)
+            .with_api_base(&config.openai_base_url),
+    ));
 
     let ingestion_pipeline =
         Arc::new(IngestionPipeline::new(db.clone(), openai_client.clone(), config).await?);

todo.md

@@ -1,9 +1,9 @@
-[] ollama and changing of openai_base_url
-[] allow changing of port the server listens to
+[] implement prompt and model choice for image processing?
+[x] ollama and changing of openai_base_url
+[x] allow changing of port the server listens to
 [] archive ingressed webpage, pdf would be easy
 [] embed surrealdb for the main binary
 [] three js graph explorer
-[] three js vector explorer
 [x] add user_id to ingress objects
 [x] admin controls re registration
 [x] allow setting of data storage folder, via envs and config