starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-01-14 14:13:25 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

Compare commits

base: starred:v0.1.4
Branches Tags
starred:main starred:development starred:benchmarks starred:hybrid-retrieval
starred:v1.0.0 starred:v0.2.7 starred:v0.2.6 starred:v0.2.5 starred:v0.2.4 starred:v0.2.3 starred:v0.2.2 starred:v0.2.1 starred:v0.2.0 starred:v0.1.4 starred:v0.1.3 starred:v0.1.2 starred:v0.1.1 starred:v0.1.0
...
compare: starred:v0.2.1
Branches Tags
starred:main starred:development starred:benchmarks starred:hybrid-retrieval
starred:v1.0.0 starred:v0.2.7 starred:v0.2.6 starred:v0.2.5 starred:v0.2.4 starred:v0.2.3 starred:v0.2.2 starred:v0.2.1 starred:v0.2.0 starred:v0.1.4 starred:v0.1.3 starred:v0.1.2 starred:v0.1.1 starred:v0.1.0

21 Commits
v0.1.4 ... v0.2.1

Author SHA1 Message Date
Per Stark
7403195df5 release: 0.2.1 
chore: remove stale todo

chore: version bump
 2025-09-24 10:25:56 +02:00
Per Stark
9faef31387 fix: json response in api to work with ios shortcuts 
fix corrected lockfile
 2025-09-23 22:01:58 +02:00
Per Stark
110f7b8a8f no graph screenshot in readme, too much image 2025-09-23 09:08:17 +02:00
Per Stark
f343005af8 release: 0.2.0 
Includes:
- Design overhaul
- New graph visualization
- Statistics on dashboard
- Supports audio files for ingestion
- Pagination for entities and content
- Improved performance
- Improved support for running as a systemd-service

version bump
 2025-09-23 09:06:59 +02:00
Per Stark
e1d98b0c35 fix: filter improvement 2025-09-23 08:59:57 +02:00
Per Stark
c12d00edaa feat: pagination for entities and knowledge pages 2025-09-22 20:54:30 +02:00
Per Stark
903585bfef fix: improved performance by truncating not displayed text 2025-09-22 15:37:55 +02:00
Per Stark
f592eb7200 fix: improved db schema strictness 
fix
 2025-09-22 15:37:22 +02:00
Per Stark
c2839f8db3 fix: somewhat improved concurrency 
limiting edge creation to sequential due to surrealdb
 2025-09-21 11:57:39 +02:00
Per Stark
9a7c57cb19 fix: contrast on code blocks 2025-09-21 10:43:57 +02:00
Per Stark
fe5143cd7f design: additional design improvements 
chore: remove stale comment
 2025-09-20 22:09:37 +02:00
Per Stark
3f774302c7 fix: dark mode prose text color 2025-09-17 14:48:38 +02:00
Per Stark
6ea51095e8 design: neobrutalist_theme into main 2025-09-17 10:01:20 +02:00
Per Stark
62d909bb7e refactor: merge new storage backend into main 
This is in preparation for s3 storage support
 2025-09-14 12:22:03 +02:00
Per Stark
69954cf78e chore: clippy helps out 2025-09-06 21:00:39 +02:00
Per Stark
153efd1a98 feat: d3js instead of plotly, improved graph visualisation 2025-09-06 20:44:41 +02:00
Per Stark
fdf29bb735 fix: only count get requests as page loads 2025-09-06 11:16:12 +02:00
Per Stark
e150b476c3 feat: observability endpoints 2025-09-06 11:14:24 +02:00
Per Stark
2e076c8236 docs: clarify test suite, ask for contributions 2025-08-23 22:31:19 +02:00
Per Stark
a0632c9768 fix: write to stderr for journalctl compatability 2025-08-22 13:41:30 +02:00
Per Stark
33300d3193 feat: handles submitted audio 2025-07-29 18:39:26 +02:00
105 changed files with 3568 additions and 1544 deletions
Expand all files Collapse all files

286
Cargo.lock generated
View File

@@ -116,6 +116,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -298,6 +313,19 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "async-compression"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "977eb15ea9efd848bb8a4a1a2500347ed7f0bf794edf0dc3ddcf439f43d36b23"
dependencies = [
"compression-codecs",
"compression-core",
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "async-convert"
version = "1.0.0"
@@ -743,15 +771,6 @@ version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3"
[[package]]
name = "basic-toml"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a"
dependencies = [
"serde",
]
[[package]]
name = "bcrypt"
version = "0.15.1"
@@ -895,6 +914,27 @@ dependencies = [
"syn 2.0.101",
]
[[package]]
name = "brotli"
version = "8.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bumpalo"
version = "3.17.0"
@@ -959,6 +999,8 @@ version = "1.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0"
dependencies = [
"jobserver",
"libc",
"shlex",
]
@@ -1221,6 +1263,7 @@ dependencies = [
"axum_session_auth",
"axum_session_surreal",
"axum_typed_multipart",
"bytes",
"chrono",
"chrono-tz",
"config",
@@ -1233,6 +1276,7 @@ dependencies = [
"minijinja-autoreload",
"minijinja-contrib",
"minijinja-embed",
"object_store 0.11.2",
"reqwest",
"serde",
"serde_json",
@@ -1266,6 +1310,26 @@ dependencies = [
"uuid",
]
[[package]]
name = "compression-codecs"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64"
dependencies = [
"brotli",
"compression-core",
"flate2",
"memchr",
"zstd",
"zstd-safe",
]
[[package]]
name = "compression-core"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb"
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -1766,12 +1830,6 @@ dependencies = [
"dtoa",
]
[[package]]
name = "dyn-clone"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
[[package]]
name = "earcutr"
version = "0.4.3"
@@ -1824,16 +1882,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "erased-serde"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e004d887f51fcb9fef17317a2f3525c887d8aa3f4f50fed920816a688284a5b7"
dependencies = [
"serde",
"typeid",
]
[[package]]
name = "errno"
version = "0.3.11"
@@ -2406,6 +2454,7 @@ dependencies = [
"axum_session_auth",
"axum_session_surreal",
"axum_typed_multipart",
"chrono",
"chrono-tz",
"common",
"composite-retrieval",
@@ -2416,7 +2465,6 @@ dependencies = [
"minijinja-autoreload",
"minijinja-contrib",
"minijinja-embed",
"plotly",
"serde",
"serde_json",
"surrealdb",
@@ -2427,6 +2475,7 @@ dependencies = [
"tower-http",
"tower-serve-static",
"tracing",
"url",
]
[[package]]
@@ -2505,15 +2554,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humansize"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
dependencies = [
"libm",
]
[[package]]
name = "humantime"
version = "2.2.0"
@@ -2930,6 +2970,16 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom 0.3.2",
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.77"
@@ -3128,7 +3178,7 @@ checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "main"
version = "0.1.4"
version = "0.2.0"
dependencies = [
"anyhow",
"api-router",
@@ -3578,6 +3628,27 @@ dependencies = [
"memchr",
]
[[package]]
name = "object_store"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf"
dependencies = [
"async-trait",
"bytes",
"chrono",
"futures",
"humantime",
"itertools 0.13.0",
"parking_lot",
"percent-encoding",
"snafu",
"tokio",
"tracing",
"url",
"walkdir",
]
[[package]]
name = "object_store"
version = "0.12.0"
@@ -3936,36 +4007,6 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "plotly"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0746e9faf2b051db76470fd428cbc0db792db05346dedaae4a75b16d7be503b5"
dependencies = [
"dyn-clone",
"erased-serde",
"once_cell",
"plotly_derive",
"rand 0.8.5",
"rinja",
"serde",
"serde_json",
"serde_repr",
"serde_with",
]
[[package]]
name = "plotly_derive"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d683930282f098b9f524e2596e3e63483507ac499231c96127fcb166bc05d26"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn 2.0.101",
]
[[package]]
name = "polyval"
version = "0.6.2"
@@ -4503,49 +4544,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rinja"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5"
dependencies = [
"humansize",
"itoa",
"percent-encoding",
"rinja_derive",
"serde",
"serde_json",
]
[[package]]
name = "rinja_derive"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b"
dependencies = [
"basic-toml",
"memchr",
"mime",
"mime_guess",
"proc-macro2",
"quote",
"rinja_parser",
"rustc-hash",
"serde",
"syn 2.0.101",
]
[[package]]
name = "rinja_parser"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610"
dependencies = [
"memchr",
"nom 7.1.3",
"serde",
]
[[package]]
name = "rkyv"
version = "0.7.45"
@@ -4996,17 +4994,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.101",
]
[[package]]
name = "serde_spanned"
version = "0.6.8"
@@ -5161,6 +5148,27 @@ dependencies = [
"serde",
]
[[package]]
name = "snafu"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2"
dependencies = [
"snafu-derive",
]
[[package]]
name = "snafu-derive"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.101",
]
[[package]]
name = "snap"
version = "1.1.1"
@@ -5390,7 +5398,7 @@ dependencies = [
"ndarray-stats",
"num-traits",
"num_cpus",
"object_store",
"object_store 0.12.0",
"parking_lot",
"pbkdf2",
"pharos",
@@ -5913,8 +5921,10 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697"
dependencies = [
"async-compression",
"bitflags 2.9.0",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
@@ -6098,12 +6108,6 @@ dependencies = [
"utf-8",
]
[[package]]
name = "typeid"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
[[package]]
name = "typenum"
version = "1.18.0"
@@ -7007,3 +7011,31 @@ dependencies = [
"quote",
"syn 2.0.101",
]
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]

5
Cargo.toml
View File

@@ -34,7 +34,6 @@ minijinja-autoreload = "2.5.0"
minijinja-contrib = { version = "2.6.0", features = ["datetime", "timezone"] }
minijinja-embed = { version = "2.8.0" }
minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
plotly = "0.12.1"
reqwest = {version = "0.12.12", features = ["charset", "json"]}
serde_json = "1.0.128"
serde = { version = "1", features = ["derive"] }
@@ -46,7 +45,7 @@ text-splitter = "0.18.1"
thiserror = "1.0.63"
tokio-util = { version = "0.7.15", features = ["io"] }
tokio = { version = "1", features = ["full"] }
tower-http = { version = "0.6.2", features = ["fs"] }
tower-http = { version = "0.6.2", features = ["fs", "compression-full"] }
tower-serve-static = "0.1.1"
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
@@ -54,6 +53,8 @@ url = { version = "2.5.2", features = ["serde"] }
uuid = { version = "1.10.0", features = ["v4", "serde"] }
tokio-retry = "0.3.0"
base64 = "0.22.1"
object_store = { version = "0.11.2" }
bytes = "1.7.1"
[profile.dist]
inherits = "release"

34
README.md
View File

@@ -6,11 +6,11 @@
[![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
[![Latest Release](https://img.shields.io/github/v/release/perstarkse/minne?sort=semver)](https://github.com/perstarkse/minne/releases/latest)
![Screenshot](screenshot.png)
![Screenshot](screenshot-dashboard.webp)
## Demo deployment
To test *Minne* out, enter [this](https://minne-demo.stark.pub) read-only demo deployment to view and test functionality out.
To test _Minne_ out, enter [this](https://minne-demo.stark.pub) read-only demo deployment to view and test functionality out.
## The "Why" Behind Minne
@@ -22,11 +22,11 @@ While developing Minne, I discovered [KaraKeep](https://karakeep.com/) (formerly
Minne is designed to make it incredibly easy to save snippets of text, URLs, and other content (limited, pending demand). Simply send content along with a category tag. Minne then ingests this, leveraging AI to create relevant nodes and relationships within its graph database, alongside your manual categorization. This graph backend allows for discoverable connections between your pieces of knowledge.
You can converse with your knowledge base through an LLM-powered chat interface (via OpenAI compatible API, like Ollama or others). For those who like to see the bigger picture, Minne also includes an **experimental feature to visually explore your knowledge graph.**
You can converse with your knowledge base through an LLM-powered chat interface (via OpenAI compatible API, like Ollama or others). For those who like to see the bigger picture, Minne also includes an feature to visually explore your knowledge graph.
You may switch and choose between models used, and have the possiblity to change the prompts to your liking. There is since release **0.1.3** the option to change embeddings length, making it easy to test another embedding model.
You may switch and choose between models used, and have the possiblity to change the prompts to your liking. There is the option to change embeddings length, making it easy to test another embedding model.
The application is built for speed and efficiency using Rust with a Server-Side Rendered (SSR) frontend (HTMX and minimal JavaScript). It's fully responsive, offering a complete mobile interface for reading, editing, and managing your content, including the graph database itself. **PWA (Progressive Web App) support** means you can "install" Minne to your device for a native-like experience. For quick capture on the go on iOS, a [**Shortcut**](https://www.icloud.com/shortcuts/9aa960600ec14329837ba4169f57a166) makes sending content to your Minne instance a breeze.
The application is built for speed and efficiency using Rust with a Server-Side Rendered (SSR) frontend (HTMX and minimal JavaScript). It's fully responsive, offering a complete mobile interface for reading, editing, and managing your content, including the graph database itself. **PWA (Progressive Web App) support** means you can "install" Minne to your device for a native-like experience. For quick capture on the go on iOS, a [**Shortcut**](https://www.icloud.com/shortcuts/e433fbd7602f4e2eaa70dca162323477) makes sending content to your Minne instance a breeze.
Minne is open source (AGPL), self-hostable, and can be deployed flexibly: via Nix, Docker Compose, pre-built binaries, or by building from source. It can run as a single `main` binary or as separate `server` and `worker` processes for optimized resource allocation.
@@ -70,7 +70,7 @@ This is a great way to manage Minne and its SurrealDB dependency together.
1. Create a `docker-compose.yml` file:
```yaml
version: '3.8'
version: "3.8"
services:
minne:
image: ghcr.io/perstarkse/minne:latest # Pulls the latest pre-built image
@@ -88,7 +88,7 @@ This is a great way to manage Minne and its SurrealDB dependency together.
SURREALDB_DATABASE: "minne_db"
SURREALDB_NAMESPACE: "minne_ns"
OPENAI_API_KEY: "your_openai_api_key_here" # IMPORTANT: Replace with your actual key
#OPENAI_BASE_URL: "your_ollama_address" # Uncomment this and change it to override the default openai base url
#OPENAI_BASE_URL: "your_ollama_address" # Uncomment this and change it to override the default openai base url
HTTP_PORT: 3000
DATA_DIR: "/data" # Data directory inside the container
RUST_LOG: "minne=info,tower_http=info" # Example logging level
@@ -177,7 +177,7 @@ Binaries for Windows, macOS, and Linux (combined `main` version) are available o
```bash
cargo run --release --bin worker
```
The compiled binaries will be in `target/release/`.
The compiled binaries will be in `target/release/`.
## Configuration
@@ -190,7 +190,7 @@ Minne can be configured using environment variables or a `config.yaml` file plac
- `SURREALDB_PASSWORD`: Password for SurrealDB (e.g., `root_password`).
- `SURREALDB_DATABASE`: Database name in SurrealDB (e.g., `minne_db`).
- `SURREALDB_NAMESPACE`: Namespace in SurrealDB (e.g., `minne_ns`).
- `OPENAI_API_KEY`: Your API key for OpenAI (e.g., `sk-YourActualOpenAIKeyGoesHere`).
- `OPENAI_API_KEY`: Your API key for OpenAI compatible endpoint (e.g., `sk-YourActualOpenAIKeyGoesHere`).
- `HTTP_PORT`: Port for the Minne server to listen on (Default: `3000`).
**Optional Configuration:**
@@ -209,8 +209,8 @@ surrealdb_database: "minne_db"
surrealdb_namespace: "minne_ns"
openai_api_key: "sk-YourActualOpenAIKeyGoesHere"
data_dir: "./minne_app_data"
http_port: 3000
# rust_log: "info"
# http_port: 3000
```
## Application Architecture (Binaries)
@@ -229,7 +229,7 @@ Once Minne is running:
1. Access the web interface at `http://localhost:3000` (or your configured port).
1. On iOS, consider setting up the [Minne iOS Shortcut](https://www.icloud.com/shortcuts/9aa960600ec14329837ba4169f57a166) for effortless content sending. **Add the shortcut, replace the [insert_url] and the [insert_api_key] snippets**.
1. Start adding notes, URLs and explore your growing knowledge graph.
1. Add notes, URLs, **audio files**, and explore your growing knowledge graph.
1. Engage with the chat interface to query your saved content.
1. Try the experimental visual graph explorer to see connections.
@@ -257,14 +257,22 @@ Once you have configured the `OPENAI_BASE_URL` to point to your desired provider
I've developed Minne primarily for my own use, but having been in the selfhosted space for a long time, and using the efforts by others, I thought I'd share with the community. Feature requests are welcome.
The roadmap as of now is:
- Handle uploaded images wisely.
- An updated explorer of the graph database.
~~- Handle uploaded images wisely.~~
~~- An updated explorer of the graph database.~~
- A TUI frontend which opens your system default editor for improved writing and document management.
## Contributing
Contributions are welcome! Whether it's bug reports, feature suggestions, documentation improvements, or code contributions, please feel free to open an issue or submit a pull request.
## Development
Run test with
```rust
cargo test
```
There is currently a variety of unit tests for commonly used functions. Additional tests, especially integration tests would be very welcome.
## License
Minne is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0)**. See the [LICENSE](LICENSE) file for details. This means if you run a modified version of Minne as a network service, you must also offer the source code of that modified version to its users.

14
api-router/src/lib.rs
View File

@@ -6,7 +6,7 @@ use axum::{
Router,
};
use middleware_api_auth::api_auth;
use routes::{categories::get_categories, ingress::ingest_data};
use routes::{categories::get_categories, ingress::ingest_data, liveness::live, readiness::ready};
pub mod api_state;
pub mod error;
@@ -19,9 +19,17 @@ where
S: Clone + Send + Sync + 'static,
ApiState: FromRef<S>,
{
Router::new()
// Public, unauthenticated endpoints (for k8s/systemd probes)
let public = Router::new()
.route("/ready", get(ready))
.route("/live", get(live));
// Protected API endpoints (require auth)
let protected = Router::new()
.route("/ingress", post(ingest_data))
.route("/categories", get(get_categories))
.layer(DefaultBodyLimit::max(1024 * 1024 * 1024))
.route_layer(from_fn_with_state(app_state.clone(), api_auth))
.route_layer(from_fn_with_state(app_state.clone(), api_auth));
public.merge(protected)
}

7
api-router/src/routes/ingress.rs
View File

@@ -1,4 +1,4 @@
use axum::{extract::State, http::StatusCode, response::IntoResponse, Extension};
use axum::{extract::State, http::StatusCode, response::IntoResponse, Extension, Json};
use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
use common::{
error::AppError,
@@ -8,6 +8,7 @@ use common::{
},
};
use futures::{future::try_join_all, TryFutureExt};
use serde_json::json;
use tempfile::NamedTempFile;
use tracing::info;
@@ -50,7 +51,7 @@ pub async fn ingest_data(
})
.collect();
try_join_all(futures).await.map_err(AppError::from)?;
try_join_all(futures).await?;
Ok(StatusCode::OK)
Ok((StatusCode::OK, Json(json!({ "status": "success" }))))
}

7
api-router/src/routes/liveness.rs Normal file
View File

@@ -0,0 +1,7 @@
use axum::{http::StatusCode, response::IntoResponse, Json};
use serde_json::json;
/// Liveness probe: always returns 200 to indicate the process is running.
pub async fn live() -> impl IntoResponse {
(StatusCode::OK, Json(json!({"status": "ok"})))
}

2
api-router/src/routes/mod.rs
View File

@@ -1,2 +1,4 @@
pub mod categories;
pub mod ingress;
pub mod liveness;
pub mod readiness;

25
api-router/src/routes/readiness.rs Normal file
View File

@@ -0,0 +1,25 @@
use axum::{extract::State, http::StatusCode, response::IntoResponse, Json};
use serde_json::json;
use crate::api_state::ApiState;
/// Readiness probe: returns 200 if core dependencies are ready, else 503.
pub async fn ready(State(state): State<ApiState>) -> impl IntoResponse {
match state.db.client.query("RETURN true").await {
Ok(_) => (
StatusCode::OK,
Json(json!({
"status": "ok",
"checks": { "db": "ok" }
})),
),
Err(e) => (
StatusCode::SERVICE_UNAVAILABLE,
Json(json!({
"status": "error",
"checks": { "db": "fail" },
"reason": e.to_string()
})),
),
}
}

2
common/Cargo.toml
View File

@@ -39,6 +39,8 @@ url = { workspace = true }
uuid = { workspace = true }
surrealdb-migrations = { workspace = true }
tokio-retry = { workspace = true }
object_store = { workspace = true }
bytes = { workspace = true }
[features]

5
common/migrations/20250701_000000_add_voice_processing_model_to_system_settings.surql Normal file
View File

@@ -0,0 +1,5 @@
DEFINE FIELD IF NOT EXISTS voice_processing_model ON system_settings TYPE string;
UPDATE system_settings:current SET
voice_processing_model = "whisper-1"
WHERE voice_processing_model == NONE;

115
common/migrations/20250921_120004_fix_datetime_fields.surql Normal file
View File

@@ -0,0 +1,115 @@
-- Align timestamp fields with SurrealDB native datetime type.
-- User timestamps
DEFINE FIELD OVERWRITE created_at ON user FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON user FLEXIBLE;
UPDATE user SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE user SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON user TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON user TYPE datetime;
-- Text content timestamps
DEFINE FIELD OVERWRITE created_at ON text_content FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON text_content FLEXIBLE;
UPDATE text_content SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE text_content SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON text_content TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON text_content TYPE datetime;
REBUILD INDEX text_content_created_at_idx ON text_content;
-- Text chunk timestamps
DEFINE FIELD OVERWRITE created_at ON text_chunk FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON text_chunk FLEXIBLE;
UPDATE text_chunk SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE text_chunk SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON text_chunk TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON text_chunk TYPE datetime;
-- Knowledge entity timestamps
DEFINE FIELD OVERWRITE created_at ON knowledge_entity FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON knowledge_entity FLEXIBLE;
UPDATE knowledge_entity SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE knowledge_entity SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON knowledge_entity TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON knowledge_entity TYPE datetime;
REBUILD INDEX knowledge_entity_created_at_idx ON knowledge_entity;
-- Conversation timestamps
DEFINE FIELD OVERWRITE created_at ON conversation FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON conversation FLEXIBLE;
UPDATE conversation SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE conversation SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON conversation TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON conversation TYPE datetime;
REBUILD INDEX conversation_created_at_idx ON conversation;
-- Message timestamps
DEFINE FIELD OVERWRITE created_at ON message FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON message FLEXIBLE;
UPDATE message SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE message SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON message TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON message TYPE datetime;
REBUILD INDEX message_updated_at_idx ON message;
-- Ingestion task timestamps
DEFINE FIELD OVERWRITE created_at ON ingestion_task FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON ingestion_task FLEXIBLE;
UPDATE ingestion_task SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE ingestion_task SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON ingestion_task TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON ingestion_task TYPE datetime;
REBUILD INDEX idx_ingestion_task_created ON ingestion_task;
-- File timestamps
DEFINE FIELD OVERWRITE created_at ON file FLEXIBLE;
DEFINE FIELD OVERWRITE updated_at ON file FLEXIBLE;
UPDATE file SET created_at = type::datetime(created_at)
WHERE type::is::string(created_at) AND created_at != "";
UPDATE file SET updated_at = type::datetime(updated_at)
WHERE type::is::string(updated_at) AND updated_at != "";
DEFINE FIELD OVERWRITE created_at ON file TYPE datetime;
DEFINE FIELD OVERWRITE updated_at ON file TYPE datetime;

1
common/migrations/definitions/20250701_000000_add_voice_processing_model_to_system_settings.json Normal file
View File

@@ -0,0 +1 @@
{"schemas":"--- original\n+++ modified\n@@ -160,6 +160,7 @@\n DEFINE FIELD IF NOT EXISTS query_system_prompt ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS ingestion_system_prompt ON system_settings TYPE string;\n DEFINE FIELD IF NOT EXISTS image_processing_prompt ON system_settings TYPE string;\n+DEFINE FIELD IF NOT EXISTS voice_processing_model ON system_settings TYPE string;\n\n # Defines the schema for the 'text_chunk' table.\n\n","events":null}

1
common/migrations/definitions/20250921_120004_fix_datetime_fields.json Normal file
View File

@@ -0,0 +1 @@
{"schemas":"--- original\n+++ modified\n@@ -18,8 +18,8 @@\n DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;\n\n # Custom fields from the Conversation struct\n DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;\n@@ -34,8 +34,8 @@\n DEFINE TABLE IF NOT EXISTS file SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;\n\n # Custom fields from the FileInfo struct\n DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;\n@@ -54,8 +54,8 @@\n DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;\n\n DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;\n DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;\n@@ -71,8 +71,8 @@\n DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;\n\n # Custom fields from the KnowledgeEntity struct\n DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;\n@@ -102,8 +102,8 @@\n DEFINE TABLE IF NOT EXISTS message SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime;\n\n # Custom fields from the Message struct\n DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string;\n@@ -167,8 +167,8 @@\n DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;\n\n # Custom fields from the TextChunk struct\n DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;\n@@ -191,8 +191,8 @@\n DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime;\n\n # Custom fields from the TextContent struct\n DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;\n@@ -215,8 +215,8 @@\n DEFINE TABLE IF NOT EXISTS user SCHEMALESS;\n\n # Standard fields\n-DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string;\n-DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string;\n+DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime;\n+DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime;\n\n # Custom fields from the User struct\n DEFINE FIELD IF NOT EXISTS email ON user TYPE string;\n","events":null}

4
common/schemas/conversation.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS conversation SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON conversation TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON conversation TYPE datetime;
# Custom fields from the Conversation struct
DEFINE FIELD IF NOT EXISTS user_id ON conversation TYPE string;

4
common/schemas/file.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS file SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON file TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON file TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON file TYPE datetime;
# Custom fields from the FileInfo struct
DEFINE FIELD IF NOT EXISTS sha256 ON file TYPE string;

4
common/schemas/ingestion_task.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS ingestion_task SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON ingestion_task TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON ingestion_task TYPE datetime;
DEFINE FIELD IF NOT EXISTS content ON ingestion_task TYPE object;
DEFINE FIELD IF NOT EXISTS status ON ingestion_task TYPE object;

4
common/schemas/knowledge_entity.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS knowledge_entity SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON knowledge_entity TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON knowledge_entity TYPE datetime;
# Custom fields from the KnowledgeEntity struct
DEFINE FIELD IF NOT EXISTS source_id ON knowledge_entity TYPE string;

4
common/schemas/message.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS message SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON message TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON message TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON message TYPE datetime;
# Custom fields from the Message struct
DEFINE FIELD IF NOT EXISTS conversation_id ON message TYPE string;

1
common/schemas/system_settings.surql
View File

@@ -13,3 +13,4 @@ DEFINE FIELD IF NOT EXISTS embedding_dimensions ON system_settings TYPE int;
DEFINE FIELD IF NOT EXISTS query_system_prompt ON system_settings TYPE string;
DEFINE FIELD IF NOT EXISTS ingestion_system_prompt ON system_settings TYPE string;
DEFINE FIELD IF NOT EXISTS image_processing_prompt ON system_settings TYPE string;
DEFINE FIELD IF NOT EXISTS voice_processing_model ON system_settings TYPE string;

4
common/schemas/text_chunk.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS text_chunk SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON text_chunk TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON text_chunk TYPE datetime;
# Custom fields from the TextChunk struct
DEFINE FIELD IF NOT EXISTS source_id ON text_chunk TYPE string;

4
common/schemas/text_content.surql
View File

@@ -3,8 +3,8 @@
DEFINE TABLE IF NOT EXISTS text_content SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON text_content TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON text_content TYPE datetime;
# Custom fields from the TextContent struct
DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;

4
common/schemas/user.surql
View File

@@ -4,8 +4,8 @@
DEFINE TABLE IF NOT EXISTS user SCHEMALESS;
# Standard fields
DEFINE FIELD IF NOT EXISTS created_at ON user TYPE string;
DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE string;
DEFINE FIELD IF NOT EXISTS created_at ON user TYPE datetime;
DEFINE FIELD IF NOT EXISTS updated_at ON user TYPE datetime;
# Custom fields from the User struct
DEFINE FIELD IF NOT EXISTS email ON user TYPE string;

1
common/src/storage/mod.rs
View File

@@ -1,2 +1,3 @@
pub mod db;
pub mod store;
pub mod types;

283
common/src/storage/store.rs Normal file
View File

@@ -0,0 +1,283 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::{anyhow, Result as AnyResult};
use bytes::Bytes;
use futures::stream::BoxStream;
use futures::{StreamExt, TryStreamExt};
use object_store::local::LocalFileSystem;
use object_store::{path::Path as ObjPath, ObjectStore};
use crate::utils::config::{AppConfig, StorageKind};
pub type DynStore = Arc<dyn ObjectStore>;
/// Build an object store instance anchored at the given filesystem `prefix`.
///
/// - For the `Local` backend, `prefix` is the absolute directory on disk that
/// serves as the root for all object paths passed to the store.
/// - `prefix` must already exist; this function will create it if missing.
///
/// Example (Local):
/// - prefix: `/var/data`
/// - object location: `user/uuid/file.txt`
/// - absolute path: `/var/data/user/uuid/file.txt`
pub async fn build_store(prefix: &Path, cfg: &AppConfig) -> object_store::Result<DynStore> {
match cfg.storage {
StorageKind::Local => {
if !prefix.exists() {
tokio::fs::create_dir_all(prefix).await.map_err(|e| {
object_store::Error::Generic {
store: "LocalFileSystem",
source: e.into(),
}
})?;
}
let store = LocalFileSystem::new_with_prefix(prefix)?;
Ok(Arc::new(store))
}
}
}
/// Resolve the absolute base directory used for local storage from config.
///
/// If `data_dir` is relative, it is resolved against the current working directory.
pub fn resolve_base_dir(cfg: &AppConfig) -> PathBuf {
if cfg.data_dir.starts_with('/') {
PathBuf::from(&cfg.data_dir)
} else {
std::env::current_dir()
.unwrap_or_else(|_| PathBuf::from("."))
.join(&cfg.data_dir)
}
}
/// Build an object store rooted at the configured data directory.
///
/// This is the recommended way to obtain a store for logical object operations
/// such as `put_bytes_at`, `get_bytes_at`, and `delete_prefix_at`.
pub async fn build_store_root(cfg: &AppConfig) -> object_store::Result<DynStore> {
let base = resolve_base_dir(cfg);
build_store(&base, cfg).await
}
/// Write bytes to `file_name` within a filesystem `prefix` using the configured store.
///
/// Prefer [`put_bytes_at`] for location-based writes that do not need to compute
/// a separate filesystem prefix.
pub async fn put_bytes(
prefix: &Path,
file_name: &str,
data: Bytes,
cfg: &AppConfig,
) -> object_store::Result<()> {
let store = build_store(prefix, cfg).await?;
let payload = object_store::PutPayload::from_bytes(data);
store.put(&ObjPath::from(file_name), payload).await?;
Ok(())
}
/// Write bytes to the provided logical object `location`, e.g. `"user/uuid/file"`.
///
/// The store root is taken from `AppConfig::data_dir` for the local backend.
/// This performs an atomic write as guaranteed by `object_store`.
pub async fn put_bytes_at(
location: &str,
data: Bytes,
cfg: &AppConfig,
) -> object_store::Result<()> {
let store = build_store_root(cfg).await?;
let payload = object_store::PutPayload::from_bytes(data);
store.put(&ObjPath::from(location), payload).await?;
Ok(())
}
/// Read bytes from `file_name` within a filesystem `prefix` using the configured store.
///
/// Prefer [`get_bytes_at`] for location-based reads.
pub async fn get_bytes(
prefix: &Path,
file_name: &str,
cfg: &AppConfig,
) -> object_store::Result<Bytes> {
let store = build_store(prefix, cfg).await?;
let r = store.get(&ObjPath::from(file_name)).await?;
let b = r.bytes().await?;
Ok(b)
}
/// Read bytes from the provided logical object `location`.
///
/// Returns the full contents buffered in memory.
pub async fn get_bytes_at(location: &str, cfg: &AppConfig) -> object_store::Result<Bytes> {
let store = build_store_root(cfg).await?;
let r = store.get(&ObjPath::from(location)).await?;
r.bytes().await
}
/// Get a streaming body for the provided logical object `location`.
///
/// Returns a fallible `BoxStream` of `Bytes`, suitable for use with
/// `axum::body::Body::from_stream` to stream responses without buffering.
pub async fn get_stream_at(
location: &str,
cfg: &AppConfig,
) -> object_store::Result<BoxStream<'static, object_store::Result<Bytes>>> {
let store = build_store_root(cfg).await?;
let r = store.get(&ObjPath::from(location)).await?;
Ok(r.into_stream())
}
/// Delete all objects below the provided filesystem `prefix`.
///
/// This is a low-level variant for when a dedicated on-disk prefix is used for a
/// particular object grouping. Prefer [`delete_prefix_at`] for location-based stores.
pub async fn delete_prefix(prefix: &Path, cfg: &AppConfig) -> object_store::Result<()> {
let store = build_store(prefix, cfg).await?;
// list everything and delete
let locations = store.list(None).map_ok(|m| m.location).boxed();
store
.delete_stream(locations)
.try_collect::<Vec<_>>()
.await?;
// Best effort remove the directory itself
if tokio::fs::try_exists(prefix).await.unwrap_or(false) {
let _ = tokio::fs::remove_dir_all(prefix).await;
}
Ok(())
}
/// Delete all objects below the provided logical object `prefix`, e.g. `"user/uuid/"`.
///
/// After deleting, attempts a best-effort cleanup of the now-empty directory on disk
/// when using the local backend.
pub async fn delete_prefix_at(prefix: &str, cfg: &AppConfig) -> object_store::Result<()> {
let store = build_store_root(cfg).await?;
let prefix_path = ObjPath::from(prefix);
let locations = store
.list(Some(&prefix_path))
.map_ok(|m| m.location)
.boxed();
store
.delete_stream(locations)
.try_collect::<Vec<_>>()
.await?;
// Best effort remove empty directory on disk for local storage
let base_dir = resolve_base_dir(cfg).join(prefix);
if tokio::fs::try_exists(&base_dir).await.unwrap_or(false) {
let _ = tokio::fs::remove_dir_all(&base_dir).await;
}
Ok(())
}
/// Split an absolute filesystem path into `(parent_dir, file_name)`.
pub fn split_abs_path(path: &str) -> AnyResult<(PathBuf, String)> {
let pb = PathBuf::from(path);
let parent = pb
.parent()
.ok_or_else(|| anyhow!("Path has no parent: {path}"))?
.to_path_buf();
let file = pb
.file_name()
.ok_or_else(|| anyhow!("Path has no file name: {path}"))?
.to_string_lossy()
.to_string();
Ok((parent, file))
}
/// Split a logical object location `"a/b/c"` into `("a/b", "c")`.
pub fn split_object_path(path: &str) -> AnyResult<(String, String)> {
if let Some((p, f)) = path.rsplit_once('/') {
return Ok((p.to_string(), f.to_string()));
}
Err(anyhow!("Object path has no separator: {path}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::config::StorageKind;
use bytes::Bytes;
use futures::TryStreamExt;
use uuid::Uuid;
fn test_config(root: &str) -> AppConfig {
AppConfig {
openai_api_key: "test".into(),
surrealdb_address: "test".into(),
surrealdb_username: "test".into(),
surrealdb_password: "test".into(),
surrealdb_namespace: "test".into(),
surrealdb_database: "test".into(),
data_dir: root.into(),
http_port: 0,
openai_base_url: "..".into(),
storage: StorageKind::Local,
}
}
#[tokio::test]
async fn test_build_store_root_creates_base() {
let base = format!("/tmp/minne_store_test_{}", Uuid::new_v4());
let cfg = test_config(&base);
let _ = build_store_root(&cfg).await.expect("build store root");
assert!(tokio::fs::try_exists(&base).await.unwrap_or(false));
let _ = tokio::fs::remove_dir_all(&base).await;
}
#[tokio::test]
async fn test_put_get_bytes_at_and_delete_prefix_at() {
let base = format!("/tmp/minne_store_test_{}", Uuid::new_v4());
let cfg = test_config(&base);
let location_prefix = format!("{}/{}", "user1", Uuid::new_v4());
let file_name = "file.txt";
let location = format!("{}/{}", &location_prefix, file_name);
let payload = Bytes::from_static(b"hello world");
put_bytes_at(&location, payload.clone(), &cfg)
.await
.expect("put");
let got = get_bytes_at(&location, &cfg).await.expect("get");
assert_eq!(got.as_ref(), payload.as_ref());
// Delete the whole prefix and ensure retrieval fails
delete_prefix_at(&location_prefix, &cfg)
.await
.expect("delete prefix");
assert!(get_bytes_at(&location, &cfg).await.is_err());
let _ = tokio::fs::remove_dir_all(&base).await;
}
#[tokio::test]
async fn test_get_stream_at() {
let base = format!("/tmp/minne_store_test_{}", Uuid::new_v4());
let cfg = test_config(&base);
let location = format!("{}/{}/stream.bin", "user2", Uuid::new_v4());
let content = vec![7u8; 32 * 1024]; // 32KB payload
put_bytes_at(&location, Bytes::from(content.clone()), &cfg)
.await
.expect("put");
let stream = get_stream_at(&location, &cfg).await.expect("stream");
let combined: Vec<u8> = stream
.map_ok(|chunk| chunk.to_vec())
.try_fold(Vec::new(), |mut acc, mut chunk| async move {
acc.append(&mut chunk);
Ok(acc)
})
.await
.expect("collect");
assert_eq!(combined, content);
delete_prefix_at(&split_object_path(&location).unwrap().0, &cfg)
.await
.ok();
let _ = tokio::fs::remove_dir_all(&base).await;
}
}

5
common/src/storage/types/conversation.rs
View File

@@ -67,7 +67,10 @@ impl Conversation {
let _updated: Option<Self> = db
.update((Self::table_name(), id))
.patch(PatchOp::replace("/title", new_title.to_string()))
.patch(PatchOp::replace("/updated_at", Utc::now()))
.patch(PatchOp::replace(
"/updated_at",
surrealdb::Datetime::from(Utc::now()),
))
.await?;
Ok(())

235
common/src/storage/types/file_info.rs
View File

@@ -1,18 +1,21 @@
use axum_typed_multipart::FieldData;
use mime_guess::from_path;
use object_store::Error as ObjectStoreError;
use sha2::{Digest, Sha256};
use std::{
io::{BufReader, Read},
path::{Path, PathBuf},
path::Path,
};
use tempfile::NamedTempFile;
use thiserror::Error;
use tokio::fs::remove_dir_all;
use tracing::info;
use uuid::Uuid;
use crate::{
error::AppError, storage::db::SurrealDbClient, stored_object, utils::config::AppConfig,
error::AppError,
storage::{db::SurrealDbClient, store},
stored_object,
utils::config::AppConfig,
};
#[derive(Error, Debug)]
@@ -34,6 +37,9 @@ pub enum FileError {
#[error("File name missing in metadata")]
MissingFileName,
#[error("Object store error: {0}")]
ObjectStore(#[from] ObjectStoreError),
}
stored_object!(FileInfo, "file", {
@@ -83,10 +89,7 @@ impl FileInfo {
updated_at: now,
file_name,
sha256,
path: Self::persist_file(&uuid, file, &sanitized_file_name, user_id, config)
.await?
.to_string_lossy()
.into(),
path: Self::persist_file(&uuid, file, &sanitized_file_name, user_id, config).await?,
mime_type: Self::guess_mime_type(Path::new(&sanitized_file_name)),
user_id: user_id.to_string(),
};
@@ -165,7 +168,7 @@ impl FileInfo {
}
}
/// Persists the file to the filesystem under `{data_dir}/{user_id}/{uuid}/{file_name}`.
/// Persists the file under the logical location `{user_id}/{uuid}/{file_name}`.
///
/// # Arguments
/// * `uuid` - The UUID of the file.
@@ -175,43 +178,24 @@ impl FileInfo {
/// * `config` - Application configuration containing data directory path
///
/// # Returns
/// * `Result<PathBuf, FileError>` - The persisted file path or an error.
/// * `Result<String, FileError>` - The logical object location or an error.
async fn persist_file(
uuid: &Uuid,
file: NamedTempFile,
file_name: &str,
user_id: &str,
config: &AppConfig,
) -> Result<PathBuf, FileError> {
info!("Data dir: {:?}", config.data_dir);
// Convert relative path to absolute path
let base_dir = if config.data_dir.starts_with('/') {
Path::new(&config.data_dir).to_path_buf()
} else {
std::env::current_dir()
.map_err(FileError::Io)?
.join(&config.data_dir)
};
) -> Result<String, FileError> {
// Logical object location relative to the store root
let location = format!("{}/{}/{}", user_id, uuid, file_name);
info!("Persisting to object location: {}", location);
let user_dir = base_dir.join(user_id); // Create the user directory
let uuid_dir = user_dir.join(uuid.to_string()); // Create the UUID directory under the user directory
// Create the user and UUID directories if they don't exist
tokio::fs::create_dir_all(&uuid_dir)
let bytes = tokio::fs::read(file.path()).await?;
store::put_bytes_at(&location, bytes.into(), config)
.await
.map_err(FileError::Io)?;
.map_err(FileError::from)?;
// Define the final file path
let final_path = uuid_dir.join(file_name);
info!("Final path: {:?}", final_path);
// Copy the temporary file to the final path
tokio::fs::copy(file.path(), &final_path)
.await
.map_err(FileError::Io)?;
info!("Copied file to {:?}", final_path);
Ok(final_path)
Ok(location)
}
/// Retrieves a `FileInfo` by SHA256.
@@ -240,7 +224,11 @@ impl FileInfo {
///
/// # Returns
/// `Result<(), FileError>`
pub async fn delete_by_id(id: &str, db_client: &SurrealDbClient) -> Result<(), AppError> {
pub async fn delete_by_id(
id: &str,
db_client: &SurrealDbClient,
config: &AppConfig,
) -> Result<(), AppError> {
// Get the FileInfo from the database
let file_info = match db_client.get_item::<FileInfo>(id).await? {
Some(info) => info,
@@ -252,25 +240,16 @@ impl FileInfo {
}
};
// Remove the file and its parent directory
let file_path = Path::new(&file_info.path);
if file_path.exists() {
// Get the parent directory of the file
if let Some(parent_dir) = file_path.parent() {
// Remove the entire directory containing the file
remove_dir_all(parent_dir).await?;
info!("Removed directory {:?} and its contents", parent_dir);
} else {
return Err(AppError::from(FileError::FileNotFound(
"File has no parent directory".to_string(),
)));
}
} else {
return Err(AppError::from(FileError::FileNotFound(format!(
"File at path {:?} was not found",
file_path
))));
}
// Remove the object's parent prefix in the object store
let (parent_prefix, _file_name) = store::split_object_path(&file_info.path)
.map_err(|e| AppError::from(anyhow::anyhow!(e)))?;
store::delete_prefix_at(&parent_prefix, config)
.await
.map_err(|e| AppError::from(anyhow::anyhow!(e)))?;
info!(
"Removed object prefix {} and its contents via object_store",
parent_prefix
);
// Delete the FileInfo from the database
db_client.delete_item::<FileInfo>(id).await?;
@@ -298,6 +277,7 @@ impl FileInfo {
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::config::StorageKind;
use axum::http::HeaderMap;
use axum_typed_multipart::FieldMetadata;
use std::io::Write;
@@ -326,7 +306,7 @@ mod tests {
}
#[tokio::test]
async fn test_cross_filesystem_file_operations() {
async fn test_fileinfo_create_read_delete() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();
@@ -351,6 +331,7 @@ mod tests {
surrealdb_database: "test_db".to_string(),
http_port: 3000,
openai_base_url: "..".to_string(),
storage: StorageKind::Local,
};
// Test file creation
@@ -358,14 +339,11 @@ mod tests {
.await
.expect("Failed to create file across filesystems");
// Verify the file exists and has correct content
let file_path = Path::new(&file_info.path);
assert!(file_path.exists(), "File should exist at {:?}", file_path);
let file_content = tokio::fs::read_to_string(file_path)
// Verify the file exists via object_store and has correct content
let bytes = store::get_bytes_at(&file_info.path, &config)
.await
.expect("Failed to read file content");
assert_eq!(file_content, String::from_utf8_lossy(content));
.expect("Failed to read file content via object_store");
assert_eq!(bytes, content.as_slice());
// Test file reading
let retrieved = FileInfo::get_by_id(&file_info.id, &db)
@@ -375,17 +353,20 @@ mod tests {
assert_eq!(retrieved.sha256, file_info.sha256);
// Test file deletion
FileInfo::delete_by_id(&file_info.id, &db)
FileInfo::delete_by_id(&file_info.id, &db, &config)
.await
.expect("Failed to delete file");
assert!(!file_path.exists(), "File should be deleted");
assert!(
store::get_bytes_at(&file_info.path, &config).await.is_err(),
"File should be deleted"
);
// Clean up the test directory
let _ = tokio::fs::remove_dir_all(&config.data_dir).await;
}
#[tokio::test]
async fn test_cross_filesystem_duplicate_detection() {
async fn test_fileinfo_duplicate_detection() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();
@@ -410,6 +391,7 @@ mod tests {
surrealdb_database: "test_db".to_string(),
http_port: 3000,
openai_base_url: "..".to_string(),
storage: StorageKind::Local,
};
// Store the original file
@@ -433,7 +415,7 @@ mod tests {
assert_ne!(duplicate_file_info.file_name, duplicate_name);
// Clean up
FileInfo::delete_by_id(&original_file_info.id, &db)
FileInfo::delete_by_id(&original_file_info.id, &db, &config)
.await
.expect("Failed to delete file");
let _ = tokio::fs::remove_dir_all(&config.data_dir).await;
@@ -465,6 +447,7 @@ mod tests {
surrealdb_database: "test_db".to_string(),
http_port: 3000,
openai_base_url: "..".to_string(),
storage: StorageKind::Local,
};
let file_info = FileInfo::new(field_data, &db, user_id, &config).await;
@@ -478,6 +461,11 @@ mod tests {
assert_eq!(file_info.file_name, file_name);
assert!(!file_info.sha256.is_empty());
assert!(!file_info.path.is_empty());
// path should be logical: "user_id/uuid/file_name"
let parts: Vec<&str> = file_info.path.split('/').collect();
assert_eq!(parts.len(), 3);
assert_eq!(parts[0], user_id);
assert_eq!(parts[2], file_name);
assert!(file_info.mime_type.contains("text/plain"));
// Verify it's in the database
@@ -516,6 +504,7 @@ mod tests {
surrealdb_database: "test_db".to_string(),
http_port: 3000,
openai_base_url: "..".to_string(),
storage: StorageKind::Local,
};
let field_data1 = create_test_file(content, file_name);
@@ -667,50 +656,27 @@ mod tests {
.await
.expect("Failed to start in-memory surrealdb");
// Create a FileInfo instance directly (without persistence to disk)
let now = Utc::now();
let file_id = Uuid::new_v4().to_string();
// Create a temporary directory that mimics the structure we would have on disk
let base_dir = Path::new("./data");
let user_id = "test_user";
let user_dir = base_dir.join(user_id);
let uuid_dir = user_dir.join(&file_id);
tokio::fs::create_dir_all(&uuid_dir)
.await
.expect("Failed to create test directories");
// Create a test file in the directory
let test_file_path = uuid_dir.join("test_file.txt");
tokio::fs::write(&test_file_path, b"test content")
.await
.expect("Failed to write test file");
// The file path should point to our test file
let file_info = FileInfo {
id: file_id.clone(),
user_id: "user123".to_string(),
created_at: now,
updated_at: now,
sha256: "test_sha256_hash".to_string(),
path: test_file_path.to_string_lossy().to_string(),
file_name: "test_file.txt".to_string(),
mime_type: "text/plain".to_string(),
// Create and persist a test file via FileInfo::new
let user_id = "user123";
let cfg = AppConfig {
data_dir: "./data".to_string(),
openai_api_key: "".to_string(),
surrealdb_address: "".to_string(),
surrealdb_username: "".to_string(),
surrealdb_password: "".to_string(),
surrealdb_namespace: "".to_string(),
surrealdb_database: "".to_string(),
http_port: 0,
openai_base_url: "".to_string(),
storage: crate::utils::config::StorageKind::Local,
};
// Store it in the database
db.store_item(file_info.clone())
let temp = create_test_file(b"test content", "test_file.txt");
let file_info = FileInfo::new(temp, &db, user_id, &cfg)
.await
.expect("Failed to store file info");
// Verify file exists on disk
assert!(tokio::fs::try_exists(&test_file_path)
.await
.unwrap_or(false));
.expect("create file");
// Delete the file
let delete_result = FileInfo::delete_by_id(&file_id, &db).await;
let delete_result = FileInfo::delete_by_id(&file_info.id, &db, &cfg).await;
// Delete should be successful
assert!(
@@ -721,7 +687,7 @@ mod tests {
// Verify the file is removed from the database
let retrieved: Option<FileInfo> = db
.get_item(&file_id)
.get_item(&file_info.id)
.await
.expect("Failed to query database");
assert!(
@@ -729,14 +695,8 @@ mod tests {
"FileInfo should be deleted from the database"
);
// Verify directory is gone
assert!(
!tokio::fs::try_exists(&uuid_dir).await.unwrap_or(true),
"UUID directory should be deleted"
);
// Clean up test directory if it exists
let _ = tokio::fs::remove_dir_all(base_dir).await;
// Verify content no longer retrievable
assert!(store::get_bytes_at(&file_info.path, &cfg).await.is_err());
}
#[tokio::test]
@@ -749,7 +709,23 @@ mod tests {
.expect("Failed to start in-memory surrealdb");
// Try to delete a file that doesn't exist
let result = FileInfo::delete_by_id("nonexistent_id", &db).await;
let result = FileInfo::delete_by_id(
"nonexistent_id",
&db,
&AppConfig {
data_dir: "./data".to_string(),
openai_api_key: "".to_string(),
surrealdb_address: "".to_string(),
surrealdb_username: "".to_string(),
surrealdb_password: "".to_string(),
surrealdb_namespace: "".to_string(),
surrealdb_database: "".to_string(),
http_port: 0,
openai_base_url: "".to_string(),
storage: crate::utils::config::StorageKind::Local,
},
)
.await;
// Should fail with FileNotFound error
assert!(result.is_err());
@@ -828,7 +804,7 @@ mod tests {
}
#[tokio::test]
async fn test_data_directory_configuration() {
async fn test_fileinfo_persist_with_custom_root() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();
@@ -854,6 +830,7 @@ mod tests {
surrealdb_database: "test_db".to_string(),
http_port: 3000,
openai_base_url: "..".to_string(),
storage: StorageKind::Local,
};
// Test file creation
@@ -861,27 +838,17 @@ mod tests {
.await
.expect("Failed to create file in custom data directory");
// Verify the file exists in the correct location
let file_path = Path::new(&file_info.path);
assert!(file_path.exists(), "File should exist at {:?}", file_path);
// Verify the file is in the correct data directory
assert!(
file_path.starts_with(custom_data_dir),
"File should be stored in the custom data directory"
);
// Verify the file has the correct content
let file_content = tokio::fs::read_to_string(file_path)
// Verify the file has the correct content via object_store
let file_content = store::get_bytes_at(&file_info.path, &config)
.await
.expect("Failed to read file content");
assert_eq!(file_content, String::from_utf8_lossy(content));
assert_eq!(file_content.as_ref(), content);
// Test file deletion
FileInfo::delete_by_id(&file_info.id, &db)
FileInfo::delete_by_id(&file_info.id, &db, &config)
.await
.expect("Failed to delete file");
assert!(!file_path.exists(), "File should be deleted");
assert!(store::get_bytes_at(&file_info.path, &config).await.is_err());
// Clean up the test directory
let _ = tokio::fs::remove_dir_all(custom_data_dir).await;

2
common/src/storage/types/ingestion_task.rs
View File

@@ -67,7 +67,7 @@ impl IngestionTask {
.patch(PatchOp::replace("/status", status))
.patch(PatchOp::replace(
"/updated_at",
surrealdb::sql::Datetime::default(),
surrealdb::Datetime::from(Utc::now()),
))
.await?;

4
common/src/storage/types/knowledge_entity.rs
View File

@@ -103,6 +103,8 @@ impl KnowledgeEntity {
);
let embedding = generate_embedding(ai_client, &embedding_input, db_client).await?;
let now = Utc::now();
db_client
.client
.query(
@@ -117,7 +119,7 @@ impl KnowledgeEntity {
.bind(("table", Self::table_name()))
.bind(("id", id.to_string()))
.bind(("name", name.to_string()))
.bind(("updated_at", Utc::now()))
.bind(("updated_at", surrealdb::Datetime::from(now)))
.bind(("entity_type", entity_type.to_owned()))
.bind(("embedding", embedding))
.bind(("description", description.to_string()))

1
common/src/storage/types/system_settings.rs
View File

@@ -17,6 +17,7 @@ pub struct SystemSettings {
pub ingestion_system_prompt: String,
pub image_processing_model: String,
pub image_processing_prompt: String,
pub voice_processing_model: String,
}
impl StoredObject for SystemSettings {

5
common/src/storage/types/text_content.rs
View File

@@ -101,7 +101,10 @@ impl TextContent {
.patch(PatchOp::replace("/context", context))
.patch(PatchOp::replace("/category", category))
.patch(PatchOp::replace("/text", text))
.patch(PatchOp::replace("/updated_at", now))
.patch(PatchOp::replace(
"/updated_at",
surrealdb::Datetime::from(now),
))
.await?;
Ok(())

241
common/src/storage/types/user.rs
View File

@@ -1,14 +1,21 @@
use crate::{error::AppError, storage::db::SurrealDbClient, stored_object};
use async_trait::async_trait;
use axum_session_auth::Authentication;
use chrono_tz::Tz;
use surrealdb::{engine::any::Any, Surreal};
use uuid::Uuid;
use super::text_chunk::TextChunk;
use super::{
conversation::Conversation, ingestion_task::IngestionTask, knowledge_entity::KnowledgeEntity,
knowledge_relationship::KnowledgeRelationship, system_settings::SystemSettings,
conversation::Conversation,
ingestion_task::{IngestionTask, MAX_ATTEMPTS},
knowledge_entity::{KnowledgeEntity, KnowledgeEntityType},
knowledge_relationship::KnowledgeRelationship,
system_settings::SystemSettings,
text_content::TextContent,
};
use chrono::Duration;
use futures::try_join;
#[derive(Deserialize)]
pub struct CategoryResponse {
@@ -49,9 +56,6 @@ impl Authentication<User, String, Surreal<Any>> for User {
}
fn validate_timezone(input: &str) -> String {
use chrono_tz::Tz;
// Check if it's a valid IANA timezone identifier
match input.parse::<Tz>() {
Ok(_) => input.to_owned(),
Err(_) => {
@@ -61,7 +65,93 @@ fn validate_timezone(input: &str) -> String {
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct DashboardStats {
pub total_documents: i64,
pub new_documents_week: i64,
pub total_entities: i64,
pub new_entities_week: i64,
pub total_conversations: i64,
pub new_conversations_week: i64,
pub total_text_chunks: i64,
pub new_text_chunks_week: i64,
}
#[derive(Deserialize)]
struct CountResult {
count: i64,
}
impl User {
async fn count_total<T: crate::storage::types::StoredObject>(
db: &SurrealDbClient,
user_id: &str,
) -> Result<i64, AppError> {
let result: Option<CountResult> = db
.client
.query("SELECT count() as count FROM type::table($table) WHERE user_id = $user_id GROUP ALL")
.bind(("table", T::table_name()))
.bind(("user_id", user_id.to_string()))
.await?
.take(0)?;
Ok(result.map(|r| r.count).unwrap_or(0))
}
async fn count_since<T: crate::storage::types::StoredObject>(
db: &SurrealDbClient,
user_id: &str,
since: chrono::DateTime<chrono::Utc>,
) -> Result<i64, AppError> {
let result: Option<CountResult> = db
.client
.query(
"SELECT count() as count FROM type::table($table) WHERE user_id = $user_id AND created_at >= $since GROUP ALL",
)
.bind(("table", T::table_name()))
.bind(("user_id", user_id.to_string()))
.bind(("since", since))
.await?
.take(0)?;
Ok(result.map(|r| r.count).unwrap_or(0))
}
pub async fn get_dashboard_stats(
user_id: &str,
db: &SurrealDbClient,
) -> Result<DashboardStats, AppError> {
let since = chrono::Utc::now() - Duration::days(7);
let (
total_documents,
new_documents_week,
total_entities,
new_entities_week,
total_conversations,
new_conversations_week,
total_text_chunks,
new_text_chunks_week,
) = try_join!(
Self::count_total::<TextContent>(db, user_id),
Self::count_since::<TextContent>(db, user_id, since),
Self::count_total::<KnowledgeEntity>(db, user_id),
Self::count_since::<KnowledgeEntity>(db, user_id, since),
Self::count_total::<Conversation>(db, user_id),
Self::count_since::<Conversation>(db, user_id, since),
Self::count_total::<TextChunk>(db, user_id),
Self::count_since::<TextChunk>(db, user_id, since)
)?;
Ok(DashboardStats {
total_documents,
new_documents_week,
total_entities,
new_entities_week,
total_conversations,
new_conversations_week,
total_text_chunks,
new_text_chunks_week,
})
}
pub async fn create_new(
email: String,
password: String,
@@ -95,8 +185,8 @@ impl User {
.bind(("id", id))
.bind(("email", email))
.bind(("password", password))
.bind(("created_at", now))
.bind(("updated_at", now))
.bind(("created_at", surrealdb::Datetime::from(now)))
.bind(("updated_at", surrealdb::Datetime::from(now)))
.bind(("timezone", validated_tz))
.await?
.take(1)?;
@@ -266,7 +356,10 @@ impl User {
// Extract the entity types from the response
let entity_types: Vec<String> = response
.into_iter()
.map(|item| format!("{:?}", item.entity_type))
.map(|item| {
let normalized = KnowledgeEntityType::from(item.entity_type.clone());
format!("{:?}", normalized)
})
.collect();
Ok(entity_types)
@@ -444,17 +537,17 @@ impl User {
"SELECT * FROM type::table($table)
WHERE user_id = $user_id
AND (
status = 'Created'
status.name = 'Created'
OR (
status.InProgress != NONE
AND status.InProgress.attempts < $max_attempts
status.name = 'InProgress'
AND status.attempts < $max_attempts
)
)
ORDER BY created_at DESC",
)
.bind(("table", IngestionTask::table_name()))
.bind(("user_id", user_id.to_owned()))
.bind(("max_attempts", 3))
.bind(("max_attempts", MAX_ATTEMPTS))
.await?
.take(0)?;
@@ -511,6 +604,9 @@ impl User {
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::types::ingestion_payload::IngestionPayload;
use crate::storage::types::ingestion_task::{IngestionTask, IngestionTaskStatus, MAX_ATTEMPTS};
use std::collections::HashSet;
// Helper function to set up a test database with SystemSettings
async fn setup_test_db() -> SurrealDbClient {
@@ -596,6 +692,75 @@ mod tests {
assert!(nonexistent.is_err());
}
#[tokio::test]
async fn test_get_unfinished_ingestion_tasks_filters_correctly() {
let db = setup_test_db().await;
let user_id = "unfinished_user";
let other_user_id = "other_user";
let payload = IngestionPayload::Text {
text: "Test".to_string(),
context: "Context".to_string(),
category: "Category".to_string(),
user_id: user_id.to_string(),
};
let created_task = IngestionTask::new(payload.clone(), user_id.to_string()).await;
db.store_item(created_task.clone())
.await
.expect("Failed to store created task");
let mut in_progress_allowed =
IngestionTask::new(payload.clone(), user_id.to_string()).await;
in_progress_allowed.status = IngestionTaskStatus::InProgress {
attempts: 1,
last_attempt: chrono::Utc::now(),
};
db.store_item(in_progress_allowed.clone())
.await
.expect("Failed to store in-progress task");
let mut in_progress_blocked =
IngestionTask::new(payload.clone(), user_id.to_string()).await;
in_progress_blocked.status = IngestionTaskStatus::InProgress {
attempts: MAX_ATTEMPTS,
last_attempt: chrono::Utc::now(),
};
db.store_item(in_progress_blocked.clone())
.await
.expect("Failed to store blocked task");
let mut completed_task = IngestionTask::new(payload.clone(), user_id.to_string()).await;
completed_task.status = IngestionTaskStatus::Completed;
db.store_item(completed_task.clone())
.await
.expect("Failed to store completed task");
let other_payload = IngestionPayload::Text {
text: "Other".to_string(),
context: "Context".to_string(),
category: "Category".to_string(),
user_id: other_user_id.to_string(),
};
let other_task = IngestionTask::new(other_payload, other_user_id.to_string()).await;
db.store_item(other_task)
.await
.expect("Failed to store other user task");
let unfinished = User::get_unfinished_ingestion_tasks(user_id, &db)
.await
.expect("Failed to fetch unfinished tasks");
let unfinished_ids: HashSet<String> =
unfinished.iter().map(|task| task.id.clone()).collect();
assert!(unfinished_ids.contains(&created_task.id));
assert!(unfinished_ids.contains(&in_progress_allowed.id));
assert!(!unfinished_ids.contains(&in_progress_blocked.id));
assert!(!unfinished_ids.contains(&completed_task.id));
assert_eq!(unfinished_ids.len(), 2);
}
#[tokio::test]
async fn test_find_by_email() {
// Setup test database
@@ -816,4 +981,56 @@ mod tests {
let most_recent = conversations.iter().max_by_key(|c| c.created_at).unwrap();
assert_eq!(retrieved[0].id, most_recent.id);
}
#[tokio::test]
async fn test_get_latest_text_contents_returns_last_five() {
let db = setup_test_db().await;
let user_id = "latest_text_user";
let mut inserted_ids = Vec::new();
let base_time = chrono::Utc::now() - chrono::Duration::minutes(60);
for i in 0..12 {
let mut item = TextContent::new(
format!("Text {}", i),
Some(format!("Context {}", i)),
"Category".to_string(),
None,
None,
user_id.to_string(),
);
let timestamp = base_time + chrono::Duration::minutes(i);
item.created_at = timestamp;
item.updated_at = timestamp;
db.store_item(item.clone())
.await
.expect("Failed to store text content");
inserted_ids.push(item.id.clone());
}
let latest = User::get_latest_text_contents(user_id, &db)
.await
.expect("Failed to fetch latest text contents");
assert_eq!(latest.len(), 5, "Expected exactly five items");
let mut expected_ids = inserted_ids[inserted_ids.len() - 5..].to_vec();
expected_ids.reverse();
let returned_ids: Vec<String> = latest.iter().map(|item| item.id.clone()).collect();
assert_eq!(
returned_ids, expected_ids,
"Latest items did not match expectation"
);
for window in latest.windows(2) {
assert!(
window[0].created_at >= window[1].created_at,
"Results are not ordered by created_at descending"
);
}
}
}

14
common/src/utils/config.rs
View File

@@ -1,6 +1,16 @@
use config::{Config, ConfigError, Environment, File};
use serde::Deserialize;
#[derive(Clone, Deserialize, Debug)]
#[serde(rename_all = "lowercase")]
pub enum StorageKind {
Local,
}
fn default_storage_kind() -> StorageKind {
StorageKind::Local
}
#[derive(Clone, Deserialize, Debug)]
pub struct AppConfig {
pub openai_api_key: String,
@@ -14,6 +24,8 @@ pub struct AppConfig {
pub http_port: u16,
#[serde(default = "default_base_url")]
pub openai_base_url: String,
#[serde(default = "default_storage_kind")]
pub storage: StorageKind,
}
fn default_data_dir() -> String {
@@ -30,5 +42,5 @@ pub fn get_config() -> Result<AppConfig, ConfigError> {
.add_source(Environment::default())
.build()?;
Ok(config.try_deserialize()?)
config.try_deserialize()
}

2
common/src/utils/embedding.rs
View File

@@ -68,7 +68,7 @@ pub async fn generate_embedding_with_params(
let request = CreateEmbeddingRequestArgs::default()
.model(model)
.input([input])
.dimensions(dimensions as u32)
.dimensions(dimensions)
.build()?;
let response = client.embeddings().create(request).await?;

3
html-router/Cargo.toml
View File

@@ -26,11 +26,12 @@ minijinja-embed = { workspace = true }
minijinja-contrib = {workspace = true }
axum-htmx = { workspace = true }
async-stream = { workspace = true }
plotly = { workspace = true }
tower-http = { workspace = true }
chrono-tz = { workspace = true }
tower-serve-static = { workspace = true }
tokio-util = { workspace = true }
chrono = { workspace = true }
url = { workspace = true }
common = { path = "../common" }
composite-retrieval = { path = "../composite-retrieval" }

708
html-router/app.css
View File

@@ -1,29 +1,108 @@
@import 'tailwindcss' source(none);
@import 'tailwindcss';
@source './templates/**/*.html';
@plugin "daisyui" {
exclude: rootscrollbargutter;
logs: false;
themes: false;
include: [ "properties",
"scrollbar",
"rootscrolllock",
"rootcolor",
"svg",
"button",
"menu",
"navbar",
"drawer",
"modal",
"chat",
"card",
"loading",
"validator",
"fileinput",
"alert",
"swap"
];
}
@plugin "@tailwindcss/typography";
@config './tailwind.config.js';
/*
The default border color has changed to `currentColor` in Tailwind CSS v4,
so we've added these compatibility styles to make sure everything still
looks the same as it did with Tailwind CSS v3.
If we ever want to remove these styles, we need to add an explicit border
color utility to any element that depends on these defaults.
*/
@view-transition {
navigation: auto;
}
@layer base {
:root {
--nb-shadow: 4px 4px 0 0 #000;
--nb-shadow-hover: 6px 6px 0 0 #000;
}
[data-theme="light"] {
color-scheme: light;
--color-base-100: oklch(98.42% 0.012 96.42);
--color-base-200: oklch(94.52% 0.0122 96.43);
--color-base-300: oklch(90.96% 0.0125 91.53);
--color-base-content: oklch(17.76% 0 89.88);
--color-primary: oklch(20.77% 0.0398 265.75);
--color-primary-content: oklch(100% 0 89.88);
--color-secondary: oklch(54.61% 0.2152 262.88);
--color-secondary-content: oklch(100% 0 89.88);
--color-accent: oklch(72% 0.19 80);
--color-accent-content: oklch(21% 0.035 80);
--color-neutral: oklch(17.76% 0 89.88);
--color-neutral-content: oklch(96.99% 0.0013 106.42);
--color-info: oklch(60.89% 0.1109 221.72);
--color-info-content: oklch(96.99% 0.0013 106.42);
--color-success: oklch(62.71% 0.1699 149.21);
--color-success-content: oklch(96.99% 0.0013 106.42);
--color-warning: oklch(79.52% 0.1617 86.05);
--color-warning-content: oklch(17.76% 0 89.88);
--color-error: oklch(57.71% 0.2152 27.33);
--color-error-content: oklch(96.99% 0.0013 106.42);
--radius-selector: 0rem;
--radius-field: 0rem;
--radius-box: 0rem;
--size-selector: 0.25rem;
--size-field: 0.25rem;
--border: 2px;
}
[data-theme="dark"] {
color-scheme: dark;
--color-base-100: oklch(22% 0.015 255);
--color-base-200: oklch(18% 0.014 253);
--color-base-300: oklch(14% 0.012 251);
--color-base-content: oklch(97.2% 0.02 255);
--color-primary: oklch(58% 0.233 277.12);
--color-primary-content: oklch(96% 0.018 272.31);
--color-secondary: oklch(65% 0.241 354.31);
--color-secondary-content: oklch(94% 0.028 342.26);
--color-accent: oklch(78% 0.22 80);
--color-accent-content: oklch(20% 0.035 80);
--color-neutral: oklch(26% 0.02 255);
--color-neutral-content: oklch(97% 0.03 255);
--color-info: oklch(74% 0.16 232.66);
--color-info-content: oklch(29% 0.066 243.16);
--color-success: oklch(76% 0.177 163.22);
--color-success-content: oklch(37% 0.077 168.94);
--color-warning: oklch(82% 0.189 84.43);
--color-warning-content: oklch(41% 0.112 45.9);
--color-error: oklch(71% 0.194 13.43);
--color-error-content: oklch(27% 0.105 12.09);
--radius-selector: 0rem;
--radius-field: 0rem;
--radius-box: 0rem;
--size-selector: 0.25rem;
--size-field: 0.25rem;
--border: 2px;
}
body {
@apply font-satoshi;
background-color: var(--color-base-100);
color: var(--color-base-content);
font-family: 'Satoshi', sans-serif;
-webkit-font-smoothing: antialiased;
@apply selection:bg-yellow-300/40 selection:text-neutral;
}
html {
@@ -37,6 +116,581 @@
::file-selector-button {
border-color: var(--color-gray-200, currentColor);
}
.container {
padding-inline: 10px;
}
@media (min-width: 640px) {
.container {
padding-inline: 2rem;
}
}
@media (min-width: 1024px) {
.container {
padding-inline: 4rem;
}
}
@media (min-width: 1280px) {
.container {
padding-inline: 5rem;
}
}
@media (min-width: 1536px) {
.container {
padding-inline: 6rem;
}
}
.custom-scrollbar {
scrollbar-width: thin;
scrollbar-color: rgba(0, 0, 0, 0.2) transparent;
}
.custom-scrollbar::-webkit-scrollbar {
width: 4px;
}
.custom-scrollbar::-webkit-scrollbar-track {
background: transparent;
}
.custom-scrollbar::-webkit-scrollbar-thumb {
background-color: rgba(0, 0, 0, 0.2);
border-radius: 3px;
}
.hide-scrollbar {
-ms-overflow-style: none;
scrollbar-width: none;
}
.hide-scrollbar::-webkit-scrollbar {
display: none;
}
form.htmx-request {
opacity: 0.5;
}
}
/* Neobrutalist helpers influenced by Tufte principles */
@layer components {
/* Offset, hard-edge shadow; minimal ink with strong contrast */
.nb-shadow {
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
.nb-shadow-hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-card {
@apply bg-base-100 border-2 border-neutral p-4;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
.nb-card:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-panel {
@apply border-2 border-neutral;
background-color: var(--nb-panel-bg, var(--color-base-200));
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
.nb-panel:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-panel-canvas {
--nb-panel-bg: var(--color-base-100);
}
.nb-canvas {
background-color: var(--color-base-100);
}
.nb-btn {
@apply btn rounded-none border-2 border-neutral text-base-content;
--btn-color: var(--color-base-100);
--btn-fg: var(--color-base-content);
--btn-noise: none;
background-image: none;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
.nb-btn:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-link {
@apply underline underline-offset-2 decoration-neutral hover:decoration-4;
}
.nb-stat {
@apply bg-base-100 border-2 border-neutral p-5 flex flex-col gap-1;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
/* Hairline rules and quiet gridlines for Tufte feel */
.u-hairline {
@apply border-t border-neutral/20;
}
.prose-tufte {
@apply prose prose-neutral;
max-width: min(90ch, 100%);
line-height: 1.7;
}
.prose-tufte-compact {
@apply prose prose-neutral;
max-width: min(90ch, 100%);
font-size: 0.875rem;
line-height: 1.6;
}
[data-theme="dark"] .prose-tufte,
[data-theme="dark"] .prose-tufte-compact {
color: var(--color-base-content);
--tw-prose-body: var(--color-base-content);
--tw-prose-headings: var(--color-base-content);
--tw-prose-lead: rgba(255, 255, 255, 0.78);
--tw-prose-links: var(--color-accent);
--tw-prose-bold: var(--color-base-content);
--tw-prose-counters: rgba(255, 255, 255, 0.7);
--tw-prose-bullets: rgba(255, 255, 255, 0.35);
--tw-prose-hr: rgba(255, 255, 255, 0.2);
--tw-prose-quotes: var(--color-base-content);
--tw-prose-quote-borders: rgba(255, 255, 255, 0.25);
--tw-prose-captions: rgba(255, 255, 255, 0.65);
--tw-prose-code: var(--color-base-content);
--tw-prose-pre-code: inherit;
--tw-prose-pre-bg: rgba(255, 255, 255, 0.07);
--tw-prose-th-borders: rgba(255, 255, 255, 0.25);
--tw-prose-td-borders: rgba(255, 255, 255, 0.2);
}
[data-theme="dark"] .prose-tufte a,
[data-theme="dark"] .prose-tufte-compact a {
color: var(--color-accent);
}
/* Encourage a consistent card look app-wide */
.card {
@apply border-2 border-neutral rounded-none;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
.card:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
/* Input styling with good dark/light contrast */
.nb-input {
@apply rounded-none border-2 border-neutral bg-base-100 text-base-content placeholder:text-base-content/60 px-3 py-[0.5rem];
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms, border-color 150ms;
}
.nb-input:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-input:focus {
outline: none;
box-shadow: var(--nb-shadow-hover);
}
/* Select styling parallels inputs */
.nb-select {
@apply rounded-none border-2 border-neutral bg-base-100 text-base-content px-3 py-[0.5rem];
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms, border-color 150ms;
}
.nb-select:hover {
transform: translate(-1px, -1px);
box-shadow: var(--nb-shadow-hover);
}
.nb-select:focus {
outline: none;
box-shadow: var(--nb-shadow-hover);
}
/* Compact variants */
.nb-input-sm {
@apply text-sm px-2 py-[0.25rem];
}
.nb-select-sm {
@apply text-sm px-2 py-[0.25rem];
}
.nb-cta {
--btn-color: var(--color-accent);
--btn-fg: var(--color-accent-content);
--btn-noise: none;
background-image: none;
background-color: var(--color-accent);
color: var(--color-accent-content);
}
.nb-cta:hover {
background-color: var(--color-accent);
color: var(--color-accent-content);
filter: saturate(1.1) brightness(1.05);
}
/* Badges */
.nb-badge {
@apply inline-flex items-center uppercase tracking-wide text-[10px] px-2 py-0.5 bg-base-100 border-2 border-neutral rounded-none;
box-shadow: 3px 3px 0 0 #000;
}
.nb-masonry {
column-count: 1;
column-gap: 1rem;
}
.nb-masonry>* {
break-inside: avoid;
display: block;
}
@media (min-width: 768px) {
.nb-masonry {
column-count: 2;
}
}
@media (min-width: 1536px) {
.nb-masonry {
column-count: 3;
}
}
/* Chat bubbles neobrutalist */
.chat .chat-bubble {
@apply rounded-none border-2 border-neutral bg-base-100 text-neutral;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms;
}
/* Remove DaisyUI tail so our rectangle keeps clean borders/shadows */
.chat .chat-bubble::before,
.chat .chat-bubble::after {
display: none !important;
content: none !important;
}
.chat.chat-start .chat-bubble {
@apply bg-secondary text-secondary-content;
}
.chat.chat-end .chat-bubble {
@apply bg-base-100 text-neutral;
}
/* Tables */
.nb-table {
@apply w-full;
border-collapse: separate;
border-spacing: 0;
}
.nb-table thead th {
@apply uppercase tracking-wide text-xs border-b-2 border-neutral;
}
.nb-table th,
.nb-table td {
@apply p-3;
}
.nb-table tbody tr+tr td {
@apply border-t border-neutral/30;
}
.nb-table tbody tr:hover {
@apply bg-base-200/40;
}
.nb-table tbody tr:hover td:first-child {
box-shadow: inset 3px 0 0 0 #000;
}
.kg-overlay {
@apply absolute top-4 left-4 right-4 z-10 flex flex-col items-stretch gap-2;
max-width: min(420px, calc(100% - 2rem));
}
.kg-control-row {
@apply flex flex-wrap items-center gap-2;
}
.kg-control-row-primary {
@apply justify-start;
}
.kg-control-row-secondary {
@apply justify-center;
}
.kg-search-input {
@apply pl-2;
height: 2rem;
width: 100%;
max-width: 320px;
min-width: 0;
}
.kg-control-row-primary .kg-search-input {
flex: 1 1 auto;
}
.kg-search-btn {
flex: 0 0 auto;
}
.kg-toggle {
@apply transition-colors;
}
.kg-toggle-active {
--btn-color: var(--color-accent);
--btn-fg: var(--color-accent-content);
--btn-noise: none;
background-image: none;
background-color: var(--color-accent);
color: var(--color-accent-content);
}
.kg-toggle-active:hover {
background-color: var(--color-accent);
color: var(--color-accent-content);
filter: saturate(1.1) brightness(1.05);
}
@media (min-width: 768px) {
.kg-overlay {
right: auto;
max-width: none;
width: auto;
}
}
.kg-legend {
@apply absolute bottom-2 left-2 z-10 flex flex-wrap gap-4;
}
.kg-legend-card {
@apply p-2;
}
.kg-legend-heading {
@apply mb-1 text-xs opacity-70;
}
.kg-legend-row {
@apply flex items-center gap-2 text-xs;
}
/* Checkboxes */
.nb-checkbox {
@apply appearance-none inline-block align-middle rounded-none border-2 border-neutral bg-base-100;
width: 1rem;
height: 1rem;
box-shadow: var(--nb-shadow);
transition: transform 150ms, box-shadow 150ms, border-color 150ms, background-color 150ms;
background-repeat: no-repeat;
background-position: center;
background-size: 80% 80%;
cursor: pointer;
}
.nb-checkbox:hover {
transform: translate(-1px, -1px);
box-shadow: 5px 5px 0 0 #000;
}
.nb-checkbox:focus-visible {
outline: 2px solid #000;
outline-offset: 2px;
}
.nb-checkbox:active {
transform: translate(0, 0);
box-shadow: 3px 3px 0 0 #000;
}
/* Tick mark in light mode (black) */
.nb-checkbox:checked {
background-image: url("data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='3' stroke-linecap='round' stroke-linejoin='round'><polyline points='20 6 9 17 4 12'/></svg>");
}
/* Tick mark in dark mode (white) */
[data-theme="dark"] .nb-checkbox:checked {
background-image: url("data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='24' height='24' viewBox='0 0 24 24' fill='none' stroke='%23fff' stroke-width='3' stroke-linecap='round' stroke-linejoin='round'><polyline points='20 6 9 17 4 12'/></svg>");
}
/* Compact size */
.nb-checkbox-sm {
width: 0.875rem;
height: 0.875rem;
}
/* Placeholder style for smaller, quieter helper text */
.nb-input::placeholder {
font-size: 0.75rem;
letter-spacing: 0.02em;
opacity: 0.75;
}
.markdown-content {
line-height: 1.5;
word-wrap: break-word;
}
.markdown-content p {
margin-bottom: 0.75em;
}
.markdown-content p:last-child {
margin-bottom: 0;
}
.markdown-content ul,
.markdown-content ol {
margin-top: 0.5em;
margin-bottom: 0.75em;
padding-left: 2em;
}
.markdown-content li {
margin-bottom: 0.25em;
}
.markdown-content pre {
background-color: var(--color-base-200);
color: var(--color-base-content);
padding: 0.75em 1em;
border-radius: 4px;
border: 1px solid rgba(0, 0, 0, 0.08);
overflow-x: auto;
}
.markdown-content pre code {
background-color: transparent;
color: inherit;
padding: 0;
border-radius: 0;
display: block;
line-height: inherit;
}
.markdown-content :not(pre) > code {
background-color: rgba(0, 0, 0, 0.05);
color: var(--color-base-content);
padding: 0.15em 0.4em;
border-radius: 3px;
font-size: 0.9em;
}
.markdown-content table {
border-collapse: collapse;
margin: 0.75em 0;
width: 100%;
}
.markdown-content th,
.markdown-content td {
border: 1px solid rgba(0, 0, 0, 0.15);
padding: 6px 12px;
text-align: left;
}
[data-theme="dark"] .markdown-content th,
[data-theme="dark"] .markdown-content td {
border-color: rgba(255, 255, 255, 0.25);
}
.markdown-content blockquote {
border-left: 4px solid rgba(0, 0, 0, 0.15);
padding-left: 10px;
margin: 0.5em 0 0.5em 0.5em;
color: rgba(0, 0, 0, 0.6);
}
[data-theme="dark"] .markdown-content blockquote {
border-color: rgba(255, 255, 255, 0.3);
color: rgba(255, 255, 255, 0.8);
}
.markdown-content hr {
border: none;
border-top: 1px solid rgba(0, 0, 0, 0.15);
margin: 0.75em 0;
}
[data-theme="dark"] .markdown-content hr {
border-top-color: rgba(255, 255, 255, 0.2);
}
[data-theme="dark"] .markdown-content pre {
background-color: var(--color-base-200);
border-color: rgba(255, 255, 255, 0.12);
color: var(--color-base-content);
}
[data-theme="dark"] .markdown-content :not(pre) > code {
background-color: rgba(255, 255, 255, 0.12);
color: var(--color-base-content);
}
.brand-mark {
letter-spacing: 0.02em;
}
.reference-tooltip {
@apply bg-base-100 text-base-content border-2 border-neutral p-3 text-sm w-72 max-w-xs;
position: fixed;
z-index: 9999;
box-shadow: var(--nb-shadow);
}
}
/* Theme-aware placeholder contrast tweaks */
@layer base {
/* Light theme keeps default neutral tone via utilities */
[data-theme="dark"] .nb-input::placeholder,
[data-theme="dark"] .input::placeholder,
[data-theme="dark"] .textarea::placeholder,
[data-theme="dark"] textarea::placeholder,
[data-theme="dark"] input::placeholder {
color: rgba(255, 255, 255, 0.78) !important;
opacity: 0.85;
}
}
/* satoshi.css */
@@ -58,4 +712,28 @@
font-weight: 300 900;
font-style: italic;
font-display: swap;
}
}
/* Minimal override: prevent DaisyUI .menu hover bg on our nb buttons */
@layer utilities {
/* Let plain nb-btns remain transparent on hover within menus */
.menu li>.nb-btn:hover {
background-color: transparent;
}
/* Keep CTA background on hover within menus */
.menu li>.nb-cta:hover {
background-color: var(--color-accent);
color: var(--color-accent-content);
}
.toast-alert {
@apply mt-2 flex flex-col text-left gap-1;
box-shadow: var(--nb-shadow);
}
.toast-alert-title {
@apply text-lg font-bold;
}
}

2
html-router/assets/d3.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

425
html-router/assets/knowledge-graph.js Normal file
View File

@@ -0,0 +1,425 @@
// Knowledge graph renderer: interactive 2D force graph with
// zoom/pan, search, neighbor highlighting, curved links with arrows,
// responsive resize, and type/relationship legends.
(function () {
const D3_SRC = '/assets/d3.min.js';
let d3Loading = null;
function ensureD3() {
if (window.d3) return Promise.resolve();
if (d3Loading) return d3Loading;
d3Loading = new Promise((resolve, reject) => {
const s = document.createElement('script');
s.src = D3_SRC;
s.async = true;
s.onload = () => resolve();
s.onerror = () => reject(new Error('Failed to load D3'));
document.head.appendChild(s);
});
return d3Loading;
}
// Simple palettes (kept deterministic across renders)
const PALETTE_A = ['#60A5FA', '#34D399', '#F59E0B', '#A78BFA', '#F472B6', '#F87171', '#22D3EE', '#84CC16', '#FB7185'];
const PALETTE_B = ['#94A3B8', '#A3A3A3', '#9CA3AF', '#C084FC', '#FDA4AF', '#FCA5A5', '#67E8F9', '#A3E635', '#FDBA74'];
function buildMap(values) {
const unique = Array.from(new Set(values.filter(Boolean)));
const map = new Map();
unique.forEach((v, i) => map.set(v, PALETTE_A[i % PALETTE_A.length]));
return map;
}
function linkColorMap(values) {
const unique = Array.from(new Set(values.filter(Boolean)));
const map = new Map();
unique.forEach((v, i) => map.set(v, PALETTE_B[i % PALETTE_B.length]));
return map;
}
function radiusForDegree(deg) {
const d = Math.max(0, +deg || 0);
const r = 6 + Math.sqrt(d) * 3; // gentle growth
return Math.max(6, Math.min(r, 24));
}
function curvedPath(d) {
const sx = d.source.x, sy = d.source.y, tx = d.target.x, ty = d.target.y;
const dx = tx - sx, dy = ty - sy;
const dr = Math.hypot(dx, dy) * 0.7; // curve radius
const mx = (sx + tx) / 2;
const my = (sy + ty) / 2;
// Offset normal to create a consistent arc
const nx = -dy / (Math.hypot(dx, dy) || 1);
const ny = dx / (Math.hypot(dx, dy) || 1);
const cx = mx + nx * 20;
const cy = my + ny * 20;
return `M ${sx},${sy} Q ${cx},${cy} ${tx},${ty}`;
}
function buildAdjacency(nodes, links) {
const idToNode = new Map(nodes.map(n => [n.id, n]));
const neighbors = new Map();
nodes.forEach(n => neighbors.set(n.id, new Set()));
links.forEach(l => {
const s = typeof l.source === 'object' ? l.source.id : l.source;
const t = typeof l.target === 'object' ? l.target.id : l.target;
if (neighbors.has(s)) neighbors.get(s).add(t);
if (neighbors.has(t)) neighbors.get(t).add(s);
});
return { idToNode, neighbors };
}
function attachOverlay(container, { onSearch, onToggleNames, onToggleEdgeLabels, onCenter }) {
const overlay = document.createElement('div');
overlay.className = 'kg-overlay';
const primaryRow = document.createElement('div');
primaryRow.className = 'kg-control-row kg-control-row-primary';
const secondaryRow = document.createElement('div');
secondaryRow.className = 'kg-control-row kg-control-row-secondary';
// search box
const input = document.createElement('input');
input.type = 'text';
input.placeholder = 'Search nodes…';
input.className = 'nb-input kg-search-input';
input.addEventListener('keydown', (e) => {
if (e.key === 'Enter') onSearch && onSearch(input.value.trim());
});
const searchBtn = document.createElement('button');
searchBtn.className = 'nb-btn btn-xs nb-cta kg-search-btn';
searchBtn.textContent = 'Go';
searchBtn.addEventListener('click', () => onSearch && onSearch(input.value.trim()));
const namesToggle = document.createElement('button');
namesToggle.className = 'nb-btn btn-xs kg-toggle';
namesToggle.type = 'button';
namesToggle.textContent = 'Names';
namesToggle.addEventListener('click', () => onToggleNames && onToggleNames());
const labelToggle = document.createElement('button');
labelToggle.className = 'nb-btn btn-xs kg-toggle';
labelToggle.type = 'button';
labelToggle.textContent = 'Labels';
labelToggle.addEventListener('click', () => onToggleEdgeLabels && onToggleEdgeLabels());
const centerBtn = document.createElement('button');
centerBtn.className = 'nb-btn btn-xs';
centerBtn.textContent = 'Center';
centerBtn.addEventListener('click', () => onCenter && onCenter());
primaryRow.appendChild(input);
primaryRow.appendChild(searchBtn);
secondaryRow.appendChild(namesToggle);
secondaryRow.appendChild(labelToggle);
secondaryRow.appendChild(centerBtn);
overlay.appendChild(primaryRow);
overlay.appendChild(secondaryRow);
container.style.position = 'relative';
container.appendChild(overlay);
return { input, overlay, namesToggle, labelToggle };
}
function attachLegends(container, typeColor, relColor) {
const wrap = document.createElement('div');
wrap.className = 'kg-legend';
function section(title, items) {
const sec = document.createElement('div');
sec.className = 'nb-card kg-legend-card';
const h = document.createElement('div'); h.className = 'kg-legend-heading'; h.textContent = title; sec.appendChild(h);
items.forEach(([label, color]) => {
const row = document.createElement('div'); row.className = 'kg-legend-row';
const sw = document.createElement('span'); sw.style.background = color; sw.style.width = '12px'; sw.style.height = '12px'; sw.style.border = '2px solid #000';
const t = document.createElement('span'); t.textContent = label || '—';
row.appendChild(sw); row.appendChild(t); sec.appendChild(row);
});
return sec;
}
const typeItems = Array.from(typeColor.entries());
if (typeItems.length) wrap.appendChild(section('Entity Type', typeItems));
const relItems = Array.from(relColor.entries());
if (relItems.length) wrap.appendChild(section('Relationship', relItems));
container.appendChild(wrap);
return wrap;
}
async function renderKnowledgeGraph(root) {
const container = (root || document).querySelector('#knowledge-graph');
if (!container) return;
await ensureD3().catch(() => {
const err = document.createElement('div');
err.className = 'alert alert-error';
err.textContent = 'Unable to load graph library (D3).';
container.appendChild(err);
});
if (!window.d3) return;
// Clear previous render
container.innerHTML = '';
const width = container.clientWidth || 800;
const height = container.clientHeight || 600;
const et = container.dataset.entityType || '';
const cc = container.dataset.contentCategory || '';
const qs = new URLSearchParams();
if (et) qs.set('entity_type', et);
if (cc) qs.set('content_category', cc);
const url = '/knowledge/graph.json' + (qs.toString() ? ('?' + qs.toString()) : '');
let data;
try {
const res = await fetch(url, { headers: { 'Accept': 'application/json' } });
if (!res.ok) throw new Error('Failed to load graph data');
data = await res.json();
} catch (_e) {
const err = document.createElement('div');
err.className = 'alert alert-error';
err.textContent = 'Unable to load graph data.';
container.appendChild(err);
return;
}
// Color maps
const typeColor = buildMap(data.nodes.map(n => n.entity_type));
const relColor = linkColorMap(data.links.map(l => l.relationship_type));
const { neighbors } = buildAdjacency(data.nodes, data.links);
// Build overlay controls
let namesVisible = true;
let edgeLabelsVisible = true;
const togglePressedState = (button, state) => {
if (!button) return;
button.setAttribute('aria-pressed', state ? 'true' : 'false');
button.classList.toggle('kg-toggle-active', !!state);
};
const { input, namesToggle, labelToggle } = attachOverlay(container, {
onSearch: (q) => focusSearch(q),
onToggleNames: () => {
namesVisible = !namesVisible;
label.style('display', namesVisible ? null : 'none');
togglePressedState(namesToggle, namesVisible);
},
onToggleEdgeLabels: () => {
edgeLabelsVisible = !edgeLabelsVisible;
linkLabel.style('display', edgeLabelsVisible ? null : 'none');
togglePressedState(labelToggle, edgeLabelsVisible);
},
onCenter: () => zoomTo(1, [width / 2, height / 2])
});
togglePressedState(namesToggle, namesVisible);
togglePressedState(labelToggle, edgeLabelsVisible);
// SVG + zoom
const svg = d3.select(container)
.append('svg')
.attr('width', '100%')
.attr('height', height)
.attr('viewBox', [0, 0, width, height])
.attr('style', 'cursor: grab; touch-action: none; background: transparent;')
.call(d3.zoom().scaleExtent([0.25, 5]).on('zoom', (event) => {
g.attr('transform', event.transform);
}));
const g = svg.append('g');
// Defs for arrows
const defs = svg.append('defs');
const markerFor = (key, color) => {
const id = `arrow-${key.replace(/[^a-z0-9_-]/gi, '_')}`;
if (!document.getElementById(id)) {
defs.append('marker')
.attr('id', id)
.attr('viewBox', '0 -5 10 10')
.attr('refX', 16)
.attr('refY', 0)
.attr('markerWidth', 6)
.attr('markerHeight', 6)
.attr('orient', 'auto')
.append('path')
.attr('d', 'M0,-5L10,0L0,5')
.attr('fill', color);
}
return `url(#${id})`;
};
// Forces
const linkForce = d3.forceLink(data.links)
.id(d => d.id)
.distance(l => 70)
.strength(0.5);
const simulation = d3.forceSimulation(data.nodes)
.force('link', linkForce)
.force('charge', d3.forceManyBody().strength(-220))
.force('center', d3.forceCenter(width / 2, height / 2))
.force('collision', d3.forceCollide().radius(d => radiusForDegree(d.degree) + 6))
.force('y', d3.forceY(height / 2).strength(0.02))
.force('x', d3.forceX(width / 2).strength(0.02));
// Links as paths so we can curve + arrow
const link = g.append('g')
.attr('fill', 'none')
.attr('stroke-opacity', 0.7)
.selectAll('path')
.data(data.links)
.join('path')
.attr('stroke', d => relColor.get(d.relationship_type) || '#CBD5E1')
.attr('stroke-width', 1.5)
.attr('marker-end', d => markerFor(d.relationship_type || 'rel', relColor.get(d.relationship_type) || '#CBD5E1'));
// Optional edge labels (midpoint)
const linkLabel = g.append('g')
.selectAll('text')
.data(data.links)
.join('text')
.attr('font-size', 9)
.attr('fill', '#475569')
.attr('text-anchor', 'middle')
.attr('opacity', 0.7)
.text(d => d.relationship_type || '');
// Nodes
const node = g.append('g')
.attr('stroke', '#fff')
.attr('stroke-width', 1.5)
.selectAll('circle')
.data(data.nodes)
.join('circle')
.attr('r', d => radiusForDegree(d.degree))
.attr('fill', d => typeColor.get(d.entity_type) || '#94A3B8')
.attr('cursor', 'pointer')
.on('mouseenter', function (_evt, d) { setHighlight(d); })
.on('mouseleave', function () { clearHighlight(); })
.on('click', function (_evt, d) {
// pin/unpin on click
if (d.fx == null) { d.fx = d.x; d.fy = d.y; this.setAttribute('data-pinned', 'true'); }
else { d.fx = null; d.fy = null; this.removeAttribute('data-pinned'); }
})
.call(d3.drag()
.on('start', (event, d) => {
if (!event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x; d.fy = d.y;
})
.on('drag', (event, d) => { d.fx = event.x; d.fy = event.y; })
.on('end', (event, d) => { if (!event.active) simulation.alphaTarget(0); }));
node.append('title').text(d => `${d.name} • ${d.entity_type} • deg ${d.degree}`);
// Labels
const label = g.append('g')
.selectAll('text')
.data(data.nodes)
.join('text')
.text(d => d.name)
.attr('font-size', 11)
.attr('fill', '#111827')
.attr('stroke', 'white')
.attr('paint-order', 'stroke')
.attr('stroke-width', 3)
.attr('dx', d => radiusForDegree(d.degree) + 6)
.attr('dy', 4);
// Legends
attachLegends(container, typeColor, relColor);
// Highlight logic
function setHighlight(n) {
const ns = neighbors.get(n.id) || new Set();
node.attr('opacity', d => (d.id === n.id || ns.has(d.id)) ? 1 : 0.15);
label.attr('opacity', d => (d.id === n.id || ns.has(d.id)) ? 1 : 0.15);
link
.attr('stroke-opacity', d => {
const s = (typeof d.source === 'object') ? d.source.id : d.source;
const t = (typeof d.target === 'object') ? d.target.id : d.target;
return (s === n.id || t === n.id || (ns.has(s) && ns.has(t))) ? 0.9 : 0.05;
})
.attr('marker-end', d => {
const c = relColor.get(d.relationship_type) || '#CBD5E1';
return markerFor(d.relationship_type || 'rel', c);
});
linkLabel.attr('opacity', d => {
const s = (typeof d.source === 'object') ? d.source.id : d.source;
const t = (typeof d.target === 'object') ? d.target.id : d.target;
return (s === n.id || t === n.id) ? 0.9 : 0.05;
});
}
function clearHighlight() {
node.attr('opacity', 1);
label.attr('opacity', 1);
link.attr('stroke-opacity', 0.7);
linkLabel.attr('opacity', 0.7);
}
// Search + center helpers
function centerOnNode(n) {
const k = 1.5; // zoom factor
const x = n.x, y = n.y;
const transform = d3.zoomIdentity.translate(width / 2 - k * x, height / 2 - k * y).scale(k);
svg.transition().duration(350).call(zoom.transform, transform);
}
function focusSearch(query) {
if (!query) return;
const q = query.toLowerCase();
const found = data.nodes.find(n => (n.name || '').toLowerCase().includes(q));
if (found) { setHighlight(found); centerOnNode(found); }
}
// Expose zoom instance
const zoom = d3.zoom().scaleExtent([0.25, 5]).on('zoom', (event) => g.attr('transform', event.transform));
svg.call(zoom);
function zoomTo(k, center) {
const transform = d3.zoomIdentity.translate(width / 2 - k * center[0], height / 2 - k * center[1]).scale(k);
svg.transition().duration(250).call(zoom.transform, transform);
}
// Tick update
simulation.on('tick', () => {
link.attr('d', curvedPath);
node.attr('cx', d => d.x).attr('cy', d => d.y);
label.attr('x', d => d.x).attr('y', d => d.y);
linkLabel.attr('x', d => (d.source.x + d.target.x) / 2).attr('y', d => (d.source.y + d.target.y) / 2);
});
// Resize handling
const ro = new ResizeObserver(() => {
const w = container.clientWidth || width;
const h = container.clientHeight || height;
svg.attr('viewBox', [0, 0, w, h]).attr('height', h);
simulation.force('center', d3.forceCenter(w / 2, h / 2));
simulation.alpha(0.3).restart();
});
ro.observe(container);
}
function tryRender(root) {
const container = (root || document).querySelector('#knowledge-graph');
if (container) renderKnowledgeGraph(root);
}
// Expose for debugging/manual re-render
window.renderKnowledgeGraph = () => renderKnowledgeGraph(document);
// Full page load
document.addEventListener('DOMContentLoaded', () => tryRender(document));
// HTMX partial swaps
document.body.addEventListener('htmx:afterSettle', (evt) => {
tryRender(evt && evt.target ? evt.target : document);
});
})();

2
html-router/assets/style.css
View File

File diff suppressed because one or more lines are too long

29
html-router/assets/toast.js
View File

@@ -6,33 +6,31 @@
return;
}
const alert = document.createElement('div');
// Base classes for the alert
alert.className = `alert alert-${type} mt-2 shadow-md flex flex-col text-start`;
alert.className = `alert toast-alert alert-${type}`;
alert.style.opacity = '1';
alert.style.transition = 'opacity 0.5s ease-out';
// Build inner HTML based on whether title is provided
let innerHTML = '';
if (title) {
innerHTML += `<div class="font-bold text-lg">${title}</div>`; // Title element
innerHTML += `<div>${description}</div>`; // Description element
} else {
// Structure without title
innerHTML += `<span>${description}</span>`;
const titleEl = document.createElement('div');
titleEl.className = 'toast-alert-title';
titleEl.textContent = title;
alert.appendChild(titleEl);
}
alert.innerHTML = innerHTML;
const bodyEl = document.createElement(title ? 'div' : 'span');
bodyEl.textContent = description;
alert.appendChild(bodyEl);
container.appendChild(alert);
// Auto-remove after a delay
setTimeout(() => {
// Optional: Add fade-out effect
alert.style.opacity = '0';
alert.style.transition = 'opacity 0.5s ease-out';
setTimeout(() => alert.remove(), 500); // Remove after fade
}, 3000); // Start fade-out after 3 seconds
setTimeout(() => alert.remove(), 500);
}, 3000);
};
document.body.addEventListener('toast', function (event) {
console.log(event);
// Extract data from the event detail, matching the Rust payload
const detail = event.detail;
if (detail && detail.description) {
@@ -54,4 +52,3 @@
if (container) container.innerHTML = '';
});
})

3
html-router/package.json
View File

@@ -1,7 +1,6 @@
{
"name": "html-router",
"version": "1.0.0",
"main": "tailwind.config.js",
"scripts": {
"tailwind": "npx @tailwindcss/cli -i app.css -o assets/style.css -w -m"
},
@@ -14,4 +13,4 @@
"daisyui": "^5.0.12",
"tailwindcss": "^4.1.2"
}
}
}

2
html-router/src/lib.rs
View File

@@ -2,6 +2,7 @@ pub mod html_state;
pub mod middlewares;
pub mod router_factory;
pub mod routes;
pub mod utils;
use axum::{extract::FromRef, Router};
use axum_session::{Session, SessionStore};
@@ -35,5 +36,6 @@ where
.add_protected_routes(routes::content::router())
.add_protected_routes(routes::knowledge::router())
.add_protected_routes(routes::ingestion::router())
.with_compression()
.build()
}

4
html-router/src/middlewares/analytics_middleware.rs
View File

@@ -1,5 +1,6 @@
use axum::{
extract::{Request, State},
http::Method,
middleware::Next,
response::Response,
};
@@ -19,7 +20,8 @@ where
S: ProvidesDb + Clone + Send + Sync + 'static,
{
let path = request.uri().path();
if !path.starts_with("/assets") && !path.contains('.') {
// Only count visits/page loads for GET requests to non-asset, non-static paths
if request.method() == Method::GET && !path.starts_with("/assets") && !path.contains('.') {
if !session.get::<bool>("counted_visitor").unwrap_or(false) {
let _ = Analytics::increment_visitors(state.db()).await;
session.set("counted_visitor", true);

7
html-router/src/middlewares/compression.rs Normal file
View File

@@ -0,0 +1,7 @@
use tower_http::compression::CompressionLayer;
/// Provides a default compression layer that negotiates encoding based on the
/// `Accept-Encoding` header of the incoming request.
pub fn compression_layer() -> CompressionLayer {
CompressionLayer::new()
}

1
html-router/src/middlewares/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod analytics_middleware;
pub mod auth_middleware;
pub mod compression;
pub mod response_middleware;

2
html-router/src/middlewares/response_middleware.rs
View File

@@ -188,7 +188,7 @@ where
if is_htmx {
(StatusCode::OK, [(axum_htmx::HX_REDIRECT, path)], "").into_response()
} else {
Redirect::to(&path).into_response()
Redirect::to(path).into_response()
}
}
}

45
html-router/src/router_factory.rs
View File

@@ -13,7 +13,7 @@ use crate::{
html_state::HtmlState,
middlewares::{
analytics_middleware::analytics_middleware, auth_middleware::require_auth,
response_middleware::with_template_response,
compression::compression_layer, response_middleware::with_template_response,
},
};
@@ -48,6 +48,7 @@ pub struct RouterFactory<S> {
nested_protected_routes: Vec<(String, Router<S>)>,
custom_middleware: MiddleWareVecType<S>,
public_assets_config: Option<AssetsConfig>,
compression_enabled: bool,
}
struct AssetsConfig {
@@ -69,6 +70,7 @@ where
nested_protected_routes: Vec::new(),
custom_middleware: Vec::new(),
public_assets_config: None,
compression_enabled: false,
}
}
@@ -115,6 +117,12 @@ where
self
}
/// Enables response compression when building the router.
pub fn with_compression(mut self) -> Self {
self.compression_enabled = true;
self
}
pub fn build(self) -> Router<S> {
// Start with an empty router
let mut public_router = Router::new();
@@ -169,21 +177,26 @@ where
}
// Apply common middleware
router = router.layer(from_fn_with_state(
self.app_state.clone(),
analytics_middleware::<HtmlState>,
));
router = router.layer(map_response_with_state(
self.app_state.clone(),
with_template_response::<HtmlState>,
));
router = router.layer(
AuthSessionLayer::<User, String, SessionSurrealPool<Any>, Surreal<Any>>::new(Some(
self.app_state.db.client.clone(),
))
.with_config(AuthConfig::<String>::default()),
);
router = router.layer(SessionLayer::new((*self.app_state.session_store).clone()));
if self.compression_enabled {
router = router.layer(compression_layer());
}
router
.layer(from_fn_with_state(
self.app_state.clone(),
analytics_middleware::<HtmlState>,
))
.layer(map_response_with_state(
self.app_state.clone(),
with_template_response::<HtmlState>,
))
.layer(
AuthSessionLayer::<User, String, SessionSurrealPool<Any>, Surreal<Any>>::new(Some(
self.app_state.db.client.clone(),
))
.with_config(AuthConfig::<String>::default()),
)
.layer(SessionLayer::new((*self.app_state.session_store).clone()))
}
}

4
html-router/src/routes/admin/handlers.rs
View File

@@ -128,6 +128,7 @@ pub struct ModelSettingsInput {
query_model: String,
processing_model: String,
image_processing_model: String,
voice_processing_model: String,
embedding_model: String,
embedding_dimensions: Option<u32>,
}
@@ -159,6 +160,7 @@ pub async fn update_model_settings(
query_model: input.query_model,
processing_model: input.processing_model,
image_processing_model: input.image_processing_model,
voice_processing_model: input.voice_processing_model,
embedding_model: input.embedding_model,
// Use new dimensions if provided, otherwise retain the current ones.
embedding_dimensions: input
@@ -404,4 +406,4 @@ pub async fn patch_image_prompt(
settings: new_settings,
},
))
}
}

103
html-router/src/routes/content/handlers.rs
View File

@@ -3,11 +3,12 @@ use axum::{
response::IntoResponse,
Form,
};
use axum_htmx::{HxBoosted, HxRequest};
use axum_htmx::{HxBoosted, HxRequest, HxTarget};
use serde::{Deserialize, Serialize};
use common::storage::types::{
conversation::Conversation, file_info::FileInfo, text_content::TextContent, user::User, knowledge_entity::KnowledgeEntity, text_chunk::TextChunk,
conversation::Conversation, file_info::FileInfo, knowledge_entity::KnowledgeEntity,
text_chunk::TextChunk, text_content::TextContent, user::User,
};
use crate::{
@@ -16,7 +17,12 @@ use crate::{
auth_middleware::RequireUser,
response_middleware::{HtmlError, TemplateResponse},
},
utils::pagination::{paginate_items, Pagination},
utils::text_content_preview::truncate_text_contents,
};
use url::form_urlencoded;
const CONTENTS_PER_PAGE: usize = 12;
#[derive(Serialize)]
pub struct ContentPageData {
@@ -25,11 +31,20 @@ pub struct ContentPageData {
categories: Vec<String>,
selected_category: Option<String>,
conversation_archive: Vec<Conversation>,
pagination: Pagination,
page_query: String,
}
#[derive(Serialize)]
pub struct RecentTextContentData {
pub user: User,
pub text_contents: Vec<TextContent>,
}
#[derive(Deserialize)]
pub struct FilterParams {
category: Option<String>,
page: Option<usize>,
}
pub async fn show_content_page(
@@ -40,17 +55,32 @@ pub async fn show_content_page(
HxBoosted(is_boosted): HxBoosted,
) -> Result<impl IntoResponse, HtmlError> {
// Normalize empty strings to None
let has_category_param = params.category.is_some();
let category_filter = params.category.as_deref().unwrap_or("").trim();
let category_filter = params
.category
.as_ref()
.map(|c| c.trim())
.filter(|c| !c.is_empty());
// load categories and filtered/all contents
let categories = User::get_user_categories(&user.id, &state.db).await?;
let text_contents = if !category_filter.is_empty() {
User::get_text_contents_by_category(&user.id, category_filter, &state.db).await?
} else {
User::get_text_contents(&user.id, &state.db).await?
let full_contents = match category_filter {
Some(category) => {
User::get_text_contents_by_category(&user.id, category, &state.db).await?
}
None => User::get_text_contents(&user.id, &state.db).await?,
};
let (page_contents, pagination) = paginate_items(full_contents, params.page, CONTENTS_PER_PAGE);
let text_contents = truncate_text_contents(page_contents);
let page_query = category_filter
.map(|category| {
let mut serializer = form_urlencoded::Serializer::new(String::new());
serializer.append_pair("category", category);
format!("&{}", serializer.finish())
})
.unwrap_or_default();
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
let data = ContentPageData {
user,
@@ -58,19 +88,19 @@ pub async fn show_content_page(
categories,
selected_category: params.category.clone(),
conversation_archive,
pagination,
page_query,
};
if is_htmx && !is_boosted && has_category_param {
// If HTMX partial request with filter applied, return partial content list update
return Ok(TemplateResponse::new_partial(
if is_htmx && !is_boosted {
Ok(TemplateResponse::new_partial(
"content/base.html",
"main",
data,
));
))
} else {
Ok(TemplateResponse::new_template("content/base.html", data))
}
// Otherwise full page response including layout
Ok(TemplateResponse::new_template("content/base.html", data))
}
pub async fn show_text_content_edit_form(
@@ -102,13 +132,32 @@ pub async fn patch_text_content(
State(state): State<HtmlState>,
RequireUser(user): RequireUser,
Path(id): Path<String>,
HxTarget(target): HxTarget,
Form(form): Form<PatchTextContentParams>,
) -> Result<impl IntoResponse, HtmlError> {
User::get_and_validate_text_content(&id, &user.id, &state.db).await?;
TextContent::patch(&id, &form.context, &form.category, &form.text, &state.db).await?;
let text_contents = User::get_text_contents(&user.id, &state.db).await?;
if target.as_deref() == Some("latest_content_section") {
let text_contents =
truncate_text_contents(User::get_latest_text_contents(&user.id, &state.db).await?);
return Ok(TemplateResponse::new_template(
"dashboard/recent_content.html",
RecentTextContentData {
user,
text_contents,
},
));
}
let (page_contents, pagination) = paginate_items(
User::get_text_contents(&user.id, &state.db).await?,
Some(1),
CONTENTS_PER_PAGE,
);
let text_contents = truncate_text_contents(page_contents);
let categories = User::get_user_categories(&user.id, &state.db).await?;
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
@@ -121,6 +170,8 @@ pub async fn patch_text_content(
categories,
selected_category: None,
conversation_archive,
pagination,
page_query: String::new(),
},
))
}
@@ -135,7 +186,7 @@ pub async fn delete_text_content(
// If it has file info, delete that too
if let Some(file_info) = &text_content.file_info {
FileInfo::delete_by_id(&file_info.id, &state.db).await?;
FileInfo::delete_by_id(&file_info.id, &state.db, &state.config).await?;
}
// Delete related knowledge entities and text chunks
@@ -146,7 +197,12 @@ pub async fn delete_text_content(
state.db.delete_item::<TextContent>(&id).await?;
// Get updated content, categories and return the refreshed list
let text_contents = User::get_text_contents(&user.id, &state.db).await?;
let (page_contents, pagination) = paginate_items(
User::get_text_contents(&user.id, &state.db).await?,
Some(1),
CONTENTS_PER_PAGE,
);
let text_contents = truncate_text_contents(page_contents);
let categories = User::get_user_categories(&user.id, &state.db).await?;
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
@@ -158,6 +214,8 @@ pub async fn delete_text_content(
categories,
selected_category: None,
conversation_archive,
pagination,
page_query: String::new(),
},
))
}
@@ -185,13 +243,8 @@ pub async fn show_recent_content(
State(state): State<HtmlState>,
RequireUser(user): RequireUser,
) -> Result<impl IntoResponse, HtmlError> {
let text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
#[derive(Serialize)]
pub struct RecentTextContentData {
pub user: User,
pub text_contents: Vec<TextContent>,
}
let text_contents =
truncate_text_contents(User::get_latest_text_contents(&user.id, &state.db).await?);
Ok(TemplateResponse::new_template(
"dashboard/recent_content.html",

43
html-router/src/routes/index/handlers.rs
View File

@@ -4,17 +4,21 @@ use axum::{
http::{header, HeaderMap, HeaderValue, StatusCode},
response::IntoResponse,
};
use futures::try_join;
use serde::Serialize;
use tokio::{fs::File, join};
use tokio_util::io::ReaderStream;
use tokio::join;
use crate::{
html_state::HtmlState,
middlewares::{
auth_middleware::RequireUser,
response_middleware::{HtmlError, TemplateResponse},
},
utils::text_content_preview::truncate_text_contents,
AuthSessionType,
};
use common::storage::store;
use common::storage::types::user::DashboardStats;
use common::{
error::AppError,
storage::types::{
@@ -24,12 +28,11 @@ use common::{
},
};
use crate::html_state::HtmlState;
#[derive(Serialize)]
pub struct IndexPageData {
user: Option<User>,
text_contents: Vec<TextContent>,
stats: DashboardStats,
active_jobs: Vec<IngestionTask>,
conversation_archive: Vec<Conversation>,
}
@@ -42,19 +45,23 @@ pub async fn index_handler(
return Ok(TemplateResponse::redirect("/signin"));
};
let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
let (text_contents, conversation_archive, stats, active_jobs) = try_join!(
User::get_latest_text_contents(&user.id, &state.db),
User::get_user_conversations(&user.id, &state.db),
User::get_dashboard_stats(&user.id, &state.db),
User::get_unfinished_ingestion_tasks(&user.id, &state.db)
)?;
let text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
let text_contents = truncate_text_contents(text_contents);
Ok(TemplateResponse::new_template(
"dashboard/base.html",
IndexPageData {
user: Some(user),
text_contents,
active_jobs,
stats,
conversation_archive,
active_jobs,
},
))
}
@@ -77,7 +84,7 @@ pub async fn delete_text_content(
let (_res1, _res2, _res3, _res4, _res5) = join!(
async {
if let Some(file_info) = text_content.file_info {
FileInfo::delete_by_id(&file_info.id, &state.db).await
FileInfo::delete_by_id(&file_info.id, &state.db, &state.config).await
} else {
Ok(())
}
@@ -89,7 +96,8 @@ pub async fn delete_text_content(
);
// Render updated content
let latest_text_contents = User::get_latest_text_contents(&user.id, &state.db).await?;
let latest_text_contents =
truncate_text_contents(User::get_latest_text_contents(&user.id, &state.db).await?);
Ok(TemplateResponse::new_partial(
"index/signed_in/recent_content.html",
@@ -153,9 +161,8 @@ pub async fn show_active_jobs(
) -> Result<impl IntoResponse, HtmlError> {
let active_jobs = User::get_unfinished_ingestion_tasks(&user.id, &state.db).await?;
Ok(TemplateResponse::new_partial(
Ok(TemplateResponse::new_template(
"dashboard/active_jobs.html",
"active_jobs_section",
ActiveJobsData {
user: user.clone(),
active_jobs,
@@ -177,14 +184,10 @@ pub async fn serve_file(
return Ok(TemplateResponse::unauthorized().into_response());
}
let path = std::path::Path::new(&file_info.path);
let file = match File::open(path).await {
Ok(f) => f,
Err(_e) => return Ok(TemplateResponse::server_error().into_response()),
let stream = match store::get_stream_at(&file_info.path, &state.config).await {
Ok(s) => s,
Err(_) => return Ok(TemplateResponse::server_error().into_response()),
};
let stream = ReaderStream::new(file);
let body = Body::from_stream(stream);
let mut headers = HeaderMap::new();

2
html-router/src/routes/ingestion/handlers.rs
View File

@@ -83,7 +83,7 @@ pub async fn process_ingress_form(
error: String,
}
if input.content.as_ref().map_or(true, |c| c.len() < 2) && input.files.is_empty() {
if input.content.as_ref().is_none_or(|c| c.len() < 2) && input.files.is_empty() {
return Ok(TemplateResponse::new_template(
"index/signed_in/ingress_form.html",
IngressFormData {

323
html-router/src/routes/knowledge/handlers.rs
View File

@@ -1,16 +1,11 @@
use std::collections::{HashMap, VecDeque};
use std::collections::{HashMap, HashSet};
use axum::{
extract::{Path, Query, State},
response::IntoResponse,
Form,
Form, Json,
};
use axum_htmx::{HxBoosted, HxRequest};
use plotly::{
common::{Line, Marker, Mode},
layout::{Axis, LayoutScene},
Layout, Plot, Scatter3D,
};
use serde::{Deserialize, Serialize};
use common::storage::types::{
@@ -26,25 +21,32 @@ use crate::{
auth_middleware::RequireUser,
response_middleware::{HtmlError, TemplateResponse},
},
utils::pagination::{paginate_items, Pagination},
};
use url::form_urlencoded;
const KNOWLEDGE_ENTITIES_PER_PAGE: usize = 12;
#[derive(Deserialize, Default)]
pub struct FilterParams {
entity_type: Option<String>,
content_category: Option<String>,
page: Option<usize>,
}
#[derive(Serialize)]
pub struct KnowledgeBaseData {
entities: Vec<KnowledgeEntity>,
visible_entities: Vec<KnowledgeEntity>,
relationships: Vec<KnowledgeRelationship>,
user: User,
plot_html: String,
entity_types: Vec<String>,
content_categories: Vec<String>,
selected_entity_type: Option<String>,
selected_content_category: Option<String>,
conversation_archive: Vec<Conversation>,
pagination: Pagination,
page_query: String,
}
pub async fn show_knowledge_page(
@@ -54,12 +56,9 @@ pub async fn show_knowledge_page(
HxBoosted(is_boosted): HxBoosted,
Query(mut params): Query<FilterParams>,
) -> Result<impl IntoResponse, HtmlError> {
// Normalize filters
params.entity_type = params.entity_type.take().filter(|s| !s.trim().is_empty());
params.content_category = params
.content_category
.take()
.filter(|s| !s.trim().is_empty());
// Normalize filters: treat empty or "none" as no filter
params.entity_type = normalize_filter(params.entity_type.take());
params.content_category = normalize_filter(params.content_category.take());
// Load relevant data
let entity_types = User::get_entity_types(&user.id, &state.db).await?;
@@ -76,34 +75,57 @@ pub async fn show_knowledge_page(
},
};
let (visible_entities, pagination) =
paginate_items(entities.clone(), params.page, KNOWLEDGE_ENTITIES_PER_PAGE);
let page_query = {
let mut serializer = form_urlencoded::Serializer::new(String::new());
if let Some(entity_type) = params.entity_type.as_deref() {
serializer.append_pair("entity_type", entity_type);
}
if let Some(content_category) = params.content_category.as_deref() {
serializer.append_pair("content_category", content_category);
}
let encoded = serializer.finish();
if encoded.is_empty() {
String::new()
} else {
format!("&{}", encoded)
}
};
let relationships = User::get_knowledge_relationships(&user.id, &state.db).await?;
let plot_html = get_plot_html(&entities, &relationships)?;
let entity_id_set: HashSet<String> = entities.iter().map(|e| e.id.clone()).collect();
let relationships: Vec<KnowledgeRelationship> = relationships
.into_iter()
.filter(|rel| entity_id_set.contains(&rel.in_) && entity_id_set.contains(&rel.out))
.collect();
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
let kb_data = KnowledgeBaseData {
entities,
visible_entities,
relationships,
user,
plot_html,
entity_types,
content_categories,
selected_entity_type: params.entity_type.clone(),
selected_content_category: params.content_category.clone(),
conversation_archive,
pagination,
page_query,
};
// Determine response type:
// If it is an HTMX request but NOT a boosted navigation, send partial update (main block only)
// Otherwise send full page including navbar/base for direct and boosted reloads
if is_htmx && !is_boosted {
// Partial update (just main block)
Ok(TemplateResponse::new_partial(
"knowledge/base.html",
"main",
&kb_data,
))
} else {
// Full page (includes navbar etc.)
Ok(TemplateResponse::new_template(
"knowledge/base.html",
kb_data,
@@ -111,170 +133,105 @@ pub async fn show_knowledge_page(
}
}
fn get_plot_html(
entities: &[KnowledgeEntity],
relationships: &[KnowledgeRelationship],
) -> Result<String, HtmlError> {
if entities.is_empty() {
return Ok(String::new());
#[derive(Serialize)]
pub struct GraphNode {
pub id: String,
pub name: String,
pub entity_type: String,
pub degree: usize,
}
#[derive(Serialize)]
pub struct GraphLink {
pub source: String,
pub target: String,
pub relationship_type: String,
}
#[derive(Serialize)]
pub struct GraphData {
pub nodes: Vec<GraphNode>,
pub links: Vec<GraphLink>,
}
pub async fn get_knowledge_graph_json(
State(state): State<HtmlState>,
RequireUser(user): RequireUser,
Query(mut params): Query<FilterParams>,
) -> Result<impl IntoResponse, HtmlError> {
// Normalize filters: treat empty or "none" as no filter
params.entity_type = normalize_filter(params.entity_type.take());
params.content_category = normalize_filter(params.content_category.take());
// Load entities based on filters
let entities: Vec<KnowledgeEntity> = match &params.content_category {
Some(cat) => {
User::get_knowledge_entities_by_content_category(&user.id, cat, &state.db).await?
}
None => match &params.entity_type {
Some(etype) => User::get_knowledge_entities_by_type(&user.id, etype, &state.db).await?,
None => User::get_knowledge_entities(&user.id, &state.db).await?,
},
};
// All relationships for user, then filter to those whose endpoints are in the set
let relationships: Vec<KnowledgeRelationship> =
User::get_knowledge_relationships(&user.id, &state.db).await?;
let entity_ids: HashSet<String> = entities.iter().map(|e| e.id.clone()).collect();
let mut degree_count: HashMap<String, usize> = HashMap::new();
let mut links: Vec<GraphLink> = Vec::new();
for rel in relationships.iter() {
if entity_ids.contains(&rel.in_) && entity_ids.contains(&rel.out) {
// undirected counting for degree
*degree_count.entry(rel.in_.clone()).or_insert(0) += 1;
*degree_count.entry(rel.out.clone()).or_insert(0) += 1;
links.push(GraphLink {
source: rel.out.clone(),
target: rel.in_.clone(),
relationship_type: rel.metadata.relationship_type.clone(),
});
}
}
let id_to_idx: HashMap<_, _> = entities
.iter()
.enumerate()
.map(|(i, e)| (e.id.clone(), i))
let nodes: Vec<GraphNode> = entities
.into_iter()
.map(|e| GraphNode {
id: e.id.clone(),
name: e.name.clone(),
entity_type: format!("{:?}", e.entity_type),
degree: *degree_count.get(&e.id).unwrap_or(&0),
})
.collect();
// Build adjacency list
let mut graph: Vec<Vec<usize>> = vec![Vec::new(); entities.len()];
for rel in relationships {
if let (Some(&from_idx), Some(&to_idx)) = (id_to_idx.get(&rel.out), id_to_idx.get(&rel.in_))
{
graph[from_idx].push(to_idx);
graph[to_idx].push(from_idx);
}
}
// Find clusters (connected components)
let mut visited = vec![false; entities.len()];
let mut clusters: Vec<Vec<usize>> = Vec::new();
for i in 0..entities.len() {
if !visited[i] {
let mut queue = VecDeque::new();
let mut cluster = Vec::new();
queue.push_back(i);
visited[i] = true;
while let Some(node) = queue.pop_front() {
cluster.push(node);
for &nbr in &graph[node] {
if !visited[nbr] {
visited[nbr] = true;
queue.push_back(nbr);
}
}
Ok(Json(GraphData { nodes, links }))
}
// Normalize filter parameters: convert empty strings or "none" (case-insensitive) to None
fn normalize_filter(input: Option<String>) -> Option<String> {
match input {
None => None,
Some(s) => {
let trimmed = s.trim();
if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("none") {
None
} else {
Some(trim_matching_quotes(trimmed).to_string())
}
clusters.push(cluster);
}
}
// Layout params
let cluster_spacing = 20.0; // Distance between clusters
let node_spacing = 3.0; // Distance between nodes within cluster
// Arrange clusters on a Fibonacci sphere (uniform 3D positioning on unit sphere)
let cluster_count = clusters.len();
let golden_angle = std::f64::consts::PI * (3.0 - (5.0f64).sqrt());
// Will hold final positions of nodes: (x,y,z)
let mut nodes_pos = vec![(0.0f64, 0.0f64, 0.0f64); entities.len()];
for (i, cluster) in clusters.iter().enumerate() {
// Position cluster center on unit sphere scaled by cluster_spacing
let theta = golden_angle * i as f64;
let z = 1.0 - (2.0 * i as f64 + 1.0) / cluster_count as f64;
let radius = (1.0 - z * z).sqrt();
let cluster_center = (
radius * theta.cos() * cluster_spacing,
radius * theta.sin() * cluster_spacing,
z * cluster_spacing,
);
// Layout nodes within cluster as small 3D grid (cube)
// Calculate cube root to determine grid side length
let cluster_size = cluster.len();
let side_len = (cluster_size as f64).cbrt().ceil() as usize;
for (pos_in_cluster, &node_idx) in cluster.iter().enumerate() {
let x_in_cluster = (pos_in_cluster % side_len) as f64;
let y_in_cluster = ((pos_in_cluster / side_len) % side_len) as f64;
let z_in_cluster = (pos_in_cluster / (side_len * side_len)) as f64;
nodes_pos[node_idx] = (
cluster_center.0 + x_in_cluster * node_spacing,
cluster_center.1 + y_in_cluster * node_spacing,
cluster_center.2 + z_in_cluster * node_spacing,
);
}
}
let (node_x, node_y, node_z): (Vec<_>, Vec<_>, Vec<_>) = nodes_pos.iter().cloned().unzip3();
// Nodes trace
let nodes_trace = Scatter3D::new(node_x, node_y, node_z)
.mode(Mode::Markers)
.marker(Marker::new().size(8).color("#1f77b4"))
.text_array(
entities
.iter()
.map(|e| e.description.clone())
.collect::<Vec<_>>(),
)
.hover_template("Entity: %{text}<extra></extra>");
// Edges traces
let mut plot = Plot::new();
for rel in relationships {
if let (Some(&from_idx), Some(&to_idx)) = (id_to_idx.get(&rel.out), id_to_idx.get(&rel.in_))
{
let edge_x = vec![nodes_pos[from_idx].0, nodes_pos[to_idx].0];
let edge_y = vec![nodes_pos[from_idx].1, nodes_pos[to_idx].1];
let edge_z = vec![nodes_pos[from_idx].2, nodes_pos[to_idx].2];
let edge_trace = Scatter3D::new(edge_x, edge_y, edge_z)
.mode(Mode::Lines)
.line(Line::new().color("#888").width(2.0))
.hover_template(format!(
"Relationship: {}<extra></extra>",
rel.metadata.relationship_type
))
.show_legend(false);
plot.add_trace(edge_trace);
}
}
plot.add_trace(nodes_trace);
// Layout scene configuration
let layout = Layout::new()
.scene(
LayoutScene::new()
.x_axis(Axis::new().visible(false))
.y_axis(Axis::new().visible(false))
.z_axis(Axis::new().visible(false))
.camera(
plotly::layout::Camera::new()
.projection(plotly::layout::ProjectionType::Perspective.into())
.eye((2.0, 2.0, 2.0).into()),
),
)
.show_legend(false)
.paper_background_color("rgba(255,255,255,0)")
.plot_background_color("rgba(255,255,255,0)");
plot.set_layout(layout);
Ok(plot.to_html())
}
// Small utility to unzip tuple3 vectors from iterators
trait Unzip3<A, B, C> {
fn unzip3(self) -> (Vec<A>, Vec<B>, Vec<C>);
}
impl<I, A, B, C> Unzip3<A, B, C> for I
where
I: Iterator<Item = (A, B, C)>,
{
fn unzip3(self) -> (Vec<A>, Vec<B>, Vec<C>) {
let (mut va, mut vb, mut vc) = (Vec::new(), Vec::new(), Vec::new());
for (a, b, c) in self {
va.push(a);
vb.push(b);
vc.push(c);
fn trim_matching_quotes(value: &str) -> &str {
let bytes = value.as_bytes();
if bytes.len() >= 2 {
let first = bytes[0];
let last = bytes[bytes.len() - 1];
if (first == b'"' && last == b'"') || (first == b'\'' && last == b'\'') {
return &value[1..value.len() - 1];
}
(va, vb, vc)
}
value
}
pub async fn show_edit_knowledge_entity_form(
@@ -318,12 +275,14 @@ pub struct PatchKnowledgeEntityParams {
#[derive(Serialize)]
pub struct EntityListData {
entities: Vec<KnowledgeEntity>,
visible_entities: Vec<KnowledgeEntity>,
pagination: Pagination,
user: User,
entity_types: Vec<String>,
content_categories: Vec<String>,
selected_entity_type: Option<String>,
selected_content_category: Option<String>,
page_query: String,
}
pub async fn patch_knowledge_entity(
@@ -348,7 +307,11 @@ pub async fn patch_knowledge_entity(
.await?;
// Get updated list of entities
let entities = User::get_knowledge_entities(&user.id, &state.db).await?;
let (visible_entities, pagination) = paginate_items(
User::get_knowledge_entities(&user.id, &state.db).await?,
Some(1),
KNOWLEDGE_ENTITIES_PER_PAGE,
);
// Get entity types
let entity_types = User::get_entity_types(&user.id, &state.db).await?;
@@ -360,12 +323,14 @@ pub async fn patch_knowledge_entity(
Ok(TemplateResponse::new_template(
"knowledge/entity_list.html",
EntityListData {
entities,
visible_entities,
pagination,
user,
entity_types,
content_categories,
selected_entity_type: None,
selected_content_category: None,
page_query: String::new(),
},
))
}
@@ -382,7 +347,11 @@ pub async fn delete_knowledge_entity(
state.db.delete_item::<KnowledgeEntity>(&id).await?;
// Get updated list of entities
let entities = User::get_knowledge_entities(&user.id, &state.db).await?;
let (visible_entities, pagination) = paginate_items(
User::get_knowledge_entities(&user.id, &state.db).await?,
Some(1),
KNOWLEDGE_ENTITIES_PER_PAGE,
);
// Get entity types
let entity_types = User::get_entity_types(&user.id, &state.db).await?;
@@ -393,12 +362,14 @@ pub async fn delete_knowledge_entity(
Ok(TemplateResponse::new_template(
"knowledge/entity_list.html",
EntityListData {
entities,
visible_entities,
pagination,
user,
entity_types,
content_categories,
selected_entity_type: None,
selected_content_category: None,
page_query: String::new(),
},
))
}

6
html-router/src/routes/knowledge/mod.rs
View File

@@ -6,8 +6,9 @@ use axum::{
Router,
};
use handlers::{
delete_knowledge_entity, delete_knowledge_relationship, patch_knowledge_entity,
save_knowledge_relationship, show_edit_knowledge_entity_form, show_knowledge_page,
delete_knowledge_entity, delete_knowledge_relationship, get_knowledge_graph_json,
patch_knowledge_entity, save_knowledge_relationship, show_edit_knowledge_entity_form,
show_knowledge_page,
};
use crate::html_state::HtmlState;
@@ -19,6 +20,7 @@ where
{
Router::new()
.route("/knowledge", get(show_knowledge_page))
.route("/knowledge/graph.json", get(get_knowledge_graph_json))
.route(
"/knowledge-entity/{id}",
get(show_edit_knowledge_entity_form)

4
html-router/src/routes/search/handlers.rs
View File

@@ -5,6 +5,7 @@ use axum::{
response::IntoResponse,
};
use common::storage::types::{
conversation::Conversation,
text_content::{TextContent, TextContentSearchResult},
user::User,
};
@@ -47,7 +48,9 @@ pub async fn search_result_handler(
search_result: Vec<TextContentSearchResult>,
query_param: String,
user: User,
conversation_archive: Vec<Conversation>,
}
let conversation_archive = User::get_user_conversations(&user.id, &state.db).await?;
let (search_results_for_template, final_query_param_for_template) =
if let Some(actual_query) = params.query {
@@ -72,6 +75,7 @@ pub async fn search_result_handler(
search_result: search_results_for_template,
query_param: final_query_param_for_template,
user,
conversation_archive,
},
))
}

2
html-router/src/utils/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod pagination;
pub mod text_content_preview;

144
html-router/src/utils/pagination.rs Normal file
View File

@@ -0,0 +1,144 @@
use serde::Serialize;
/// Metadata describing a paginated collection.
#[derive(Debug, Clone, Serialize)]
pub struct Pagination {
pub current_page: usize,
pub per_page: usize,
pub total_items: usize,
pub total_pages: usize,
pub has_previous: bool,
pub has_next: bool,
pub previous_page: Option<usize>,
pub next_page: Option<usize>,
pub start_index: usize,
pub end_index: usize,
}
impl Pagination {
pub fn new(
current_page: usize,
per_page: usize,
total_items: usize,
total_pages: usize,
page_len: usize,
) -> Self {
let has_pages = total_pages > 0;
let has_previous = has_pages && current_page > 1;
let has_next = has_pages && current_page < total_pages;
let offset = if has_pages {
per_page.saturating_mul(current_page.saturating_sub(1))
} else {
0
};
let start_index = if page_len == 0 { 0 } else { offset + 1 };
let end_index = if page_len == 0 { 0 } else { offset + page_len };
Self {
current_page,
per_page,
total_items,
total_pages,
has_previous,
has_next,
previous_page: if has_previous {
Some(current_page - 1)
} else {
None
},
next_page: if has_next {
Some(current_page + 1)
} else {
None
},
start_index,
end_index,
}
}
}
/// Returns the items for the requested page along with pagination metadata.
pub fn paginate_items<T>(
items: Vec<T>,
requested_page: Option<usize>,
per_page: usize,
) -> (Vec<T>, Pagination) {
let per_page = per_page.max(1);
let total_items = items.len();
let total_pages = if total_items == 0 {
0
} else {
((total_items - 1) / per_page) + 1
};
let mut current_page = requested_page.unwrap_or(1);
if current_page == 0 {
current_page = 1;
}
if total_pages > 0 {
current_page = current_page.min(total_pages);
} else {
current_page = 1;
}
let offset = if total_pages == 0 {
0
} else {
per_page.saturating_mul(current_page - 1)
};
let page_items: Vec<T> = items.into_iter().skip(offset).take(per_page).collect();
let page_len = page_items.len();
let pagination = Pagination::new(current_page, per_page, total_items, total_pages, page_len);
(page_items, pagination)
}
#[cfg(test)]
mod tests {
use super::paginate_items;
#[test]
fn paginates_basic_case() {
let items: Vec<_> = (1..=25).collect();
let (page, meta) = paginate_items(items, Some(2), 10);
assert_eq!(page, vec![11, 12, 13, 14, 15, 16, 17, 18, 19, 20]);
assert_eq!(meta.current_page, 2);
assert_eq!(meta.total_pages, 3);
assert!(meta.has_previous);
assert!(meta.has_next);
assert_eq!(meta.previous_page, Some(1));
assert_eq!(meta.next_page, Some(3));
assert_eq!(meta.start_index, 11);
assert_eq!(meta.end_index, 20);
}
#[test]
fn handles_empty_items() {
let items: Vec<u8> = vec![];
let (page, meta) = paginate_items(items, Some(3), 10);
assert!(page.is_empty());
assert_eq!(meta.current_page, 1);
assert_eq!(meta.total_pages, 0);
assert!(!meta.has_previous);
assert!(!meta.has_next);
assert_eq!(meta.start_index, 0);
assert_eq!(meta.end_index, 0);
}
#[test]
fn clamps_page_to_bounds() {
let items: Vec<_> = (1..=5).collect();
let (page, meta) = paginate_items(items, Some(10), 2);
assert_eq!(page, vec![5]);
assert_eq!(meta.current_page, 3);
assert_eq!(meta.total_pages, 3);
assert_eq!(meta.has_next, false);
assert_eq!(meta.has_previous, true);
assert_eq!(meta.start_index, 5);
assert_eq!(meta.end_index, 5);
}
}

35
html-router/src/utils/text_content_preview.rs Normal file
View File

@@ -0,0 +1,35 @@
use common::storage::types::text_content::TextContent;
const TEXT_PREVIEW_LENGTH: usize = 50;
fn maybe_truncate(value: &str) -> Option<String> {
let mut char_count = 0;
for (idx, _) in value.char_indices() {
if char_count == TEXT_PREVIEW_LENGTH {
return Some(value[..idx].to_string());
}
char_count += 1;
}
None
}
pub fn truncate_text_content(mut content: TextContent) -> TextContent {
if let Some(truncated) = maybe_truncate(&content.text) {
content.text = truncated;
}
if let Some(context) = content.context.as_mut() {
if let Some(truncated) = maybe_truncate(context) {
*context = truncated;
}
}
content
}
pub fn truncate_text_contents(contents: Vec<TextContent>) -> Vec<TextContent> {
contents.into_iter().map(truncate_text_content).collect()
}

33
html-router/tailwind.config.js
View File

@@ -1,33 +0,0 @@
/** @type {import('tailwindcss').Config} */
module.exports = {
content: [
'./templates/**/*',
],
theme: {
container: {
padding: {
DEFAULT: '10px',
sm: '2rem',
lg: '4rem',
xl: '5rem',
'2xl': '6rem',
},
},
extend: {
fontFamily: {
satoshi: ['Satoshi', 'sans-serif'],
},
typography: {
DEFAULT: {
css: {
maxWidth: '90ch', // Override max-width for all prose instances
},
},
},
},
},
daisyui: {
themes: ["light", "dark"],
},
}

321
html-router/templates/admin/base.html
View File

@@ -1,191 +1,156 @@
{% extends 'body_base.html' %}
{% block title %}Minne - Account{% endblock %}
{% block title %}Minne - Admin{% endblock %}
{% block main %}
<main class="container flex-grow flex flex-col mx-auto mt-4 space-y-6">
<h1 class="text-2xl font-bold mb-2">Admin Dashboard</h1>
<div class="flex justify-center grow mt-2 sm:mt-4 pb-4">
<div class="container">
<section class="mb-4">
<div class="nb-panel p-3 flex items-center justify-between">
<h1 class="text-xl font-extrabold tracking-tight">Admin Dashboard</h1>
</div>
</section>
<div class="stats stats-vertical md:stats-horizontal shadow">
<div class="stat">
<div class="stat-title font-bold">Page loads</div>
<div class="stat-value text-secondary">{{analytics.page_loads}}</div>
<div class="stat-desc">Amount of page loads</div>
</div>
<section class="mb-4">
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4">
<div class="nb-stat">
<div class="text-xs opacity-70">Page Loads</div>
<div class="text-3xl font-extrabold">{{analytics.page_loads}}</div>
<div class="text-xs opacity-60">Total page load events</div>
</div>
<div class="nb-stat">
<div class="text-xs opacity-70">Unique Visitors</div>
<div class="text-3xl font-extrabold">{{analytics.visitors}}</div>
<div class="text-xs opacity-60">Distinct users by fingerprint</div>
</div>
<div class="nb-stat">
<div class="text-xs opacity-70">Users</div>
<div class="text-3xl font-extrabold">{{users}}</div>
<div class="text-xs opacity-60">Registered accounts</div>
</div>
</div>
</section>
<div class="stat">
<div class="stat-title font-bold">Unique visitors</div>
<div class="stat-value text-primary">{{analytics.visitors}}</div>
<div class="stat-desc">Amount of unique visitors</div>
</div>
<div class="stat">
<div class="stat-title font-bold">Users</div>
<div class="stat-value text-accent">{{users}}</div>
<div class="stat-desc">Amount of registered users</div>
</div>
</div>
<!-- Settings in Fieldset -->
<div class="grid grid-cols-1 xl:grid-cols-2 gap-6">
{% block system_prompt_section %}
<div id="system_prompt_section">
<fieldset class="fieldset p-4 shadow rounded-box">
<legend class="fieldset-legend">System Prompts</legend>
<section class="grid grid-cols-1 xl:grid-cols-2 gap-4">
{% block system_prompt_section %}
<div id="system_prompt_section" class="nb-panel p-4">
<div class="text-sm font-semibold mb-3">System Prompts</div>
<div class="flex gap-2 flex-col sm:flex-row">
<button type="button" class="btn btn-primary btn-sm" hx-get="/edit-query-prompt" hx-target="#modal"
hx-swap="innerHTML">
Edit Query Prompt
</button>
<button type="button" class="btn btn-primary btn-sm" hx-get="/edit-ingestion-prompt" hx-target="#modal"
hx-swap="innerHTML">
Edit Ingestion Prompt
</button>
<button type="button" class="btn btn-primary btn-sm" hx-get="/edit-image-prompt" hx-target="#modal"
hx-swap="innerHTML">
Edit Image Prompt
</button>
<button type="button" class="nb-btn btn-sm" hx-get="/edit-query-prompt" hx-target="#modal" hx-swap="innerHTML">Edit Query Prompt</button>
<button type="button" class="nb-btn btn-sm" hx-get="/edit-ingestion-prompt" hx-target="#modal" hx-swap="innerHTML">Edit Ingestion Prompt</button>
<button type="button" class="nb-btn btn-sm" hx-get="/edit-image-prompt" hx-target="#modal" hx-swap="innerHTML">Edit Image Prompt</button>
</div>
</fieldset>
</div>
{% endblock %}
<fieldset class="fieldset p-4 shadow rounded-box">
<legend class="fieldset-legend">AI Models</legend>
{% block model_settings_form %}
<form hx-patch="/update-model-settings" hx-swap="outerHTML">
<!-- Query Model -->
<div class="form-control mb-4">
<label class="label">
<span class="label-text">Query Model</span>
</label>
<select name="query_model" class="select select-bordered w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.query_model==model.id %} selected {% endif %}>{{model.id}}
</option>
{% endfor %}
</select>
<p class="text-xs text-gray-500 mt-1">
Current used:
<span class="font-mono">{{settings.query_model}}</span>
</p>
</div>
<!-- Processing Model -->
<div class="form-control mb-4">
<label class="label">
<span class="label-text">Processing Model</span>
</label>
<select name="processing_model" class="select select-bordered w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.processing_model==model.id %} selected {% endif %}>{{model.id}}
</option>
{% endfor %}
</select>
<p class="text-xs text-gray-500 mt-1">
Current used:
<span class="font-mono">{{settings.processing_model}}</span>
</p>
</div>
<!-- Image Processing Model -->
<div class="form-control mb-4">
<label class="label">
<span class="label-text">Image Processing Model</span>
</label>
<select name="image_processing_model" class="select select-bordered w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.image_processing_model==model.id %} selected {% endif %}>
{{model.id}}
</option>
{% endfor %}
</select>
<p class="text-xs text-gray-500 mt-1">
Current used:
<span class="font-mono">{{settings.image_processing_model}}</span>
</p>
</div>
<!-- Embedding Model -->
<div class="form-control mb-4">
<label class="label">
<span class="label-text">Embedding Model</span>
</label>
<select name="embedding_model" class="select select-bordered w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.embedding_model==model.id %} selected {% endif %}>{{model.id}}
</option>
{% endfor %}
</select>
<p class="text-xs text-gray-500 mt-1">
Current used:
<span class="font-mono">{{settings.embedding_model}} ({{settings.embedding_dimensions}} dims)</span>
</p>
</div>
<!-- Embedding Dimensions (Always Visible) -->
<div class="form-control mb-4">
<label class="label" for="embedding_dimensions">
<span class="label-text">Embedding Dimensions</span>
</label>
<input type="number" id="embedding_dimensions" name="embedding_dimensions" class="input input-bordered w-full"
value="{{ settings.embedding_dimensions }}" required />
</div>
<!-- Conditional Alert -->
<div id="embedding-change-alert" role="alert" class="alert alert-warning mt-2 hidden">
<svg xmlns="http://www.w3.org/2000/svg" class="stroke-current shrink-0 h-6 w-6" fill="none"
viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-3L13.732 4c-.77-1.333-2.694-1.333-3.464 0L3.34 16c-.77 1.333.192 3 1.732 3z" />
</svg>
<span><strong>Warning:</strong> Changing dimensions will require re-creating all embeddings. Make sure you
look up what dimensions the model uses or use a model that allows specifying embedding dimensions</span>
</div>
<button type="submit" class="btn btn-primary btn-sm mt-4">Save Model Settings</button>
</form>
<script>
// Use a self-executing function to avoid polluting the global scope
// and to ensure it runs correctly after an HTMX swap.
(() => {
const dimensionInput = document.getElementById('embedding_dimensions');
const alertElement = document.getElementById('embedding-change-alert');
// The initial value is read directly from the template each time this script runs.
const initialDimensions = '{{ settings.embedding_dimensions }}';
if (dimensionInput && alertElement) {
// Use the 'input' event for immediate feedback as the user types.
dimensionInput.addEventListener('input', (event) => {
// Show alert if the current value is not the initial value. Hide it otherwise.
if (event.target.value !== initialDimensions) {
alertElement.classList.remove('hidden');
} else {
alertElement.classList.add('hidden');
}
});
}
})();
</script>
</div>
{% endblock %}
</fieldset>
<fieldset class="fieldset p-4 shadow rounded-box">
<legend class="fieldset-legend">Registration</legend>
<label class="flex gap-4 text-center">
{% block registration_status_input %}
<form hx-patch="/toggle-registrations" hx-swap="outerHTML" hx-trigger="change">
<input name="registration_open" type="checkbox" class="checkbox" {% if settings.registrations_enabled
%}checked{% endif %} />
<div class="nb-panel p-4">
<div class="text-sm font-semibold mb-3">AI Models</div>
{% block model_settings_form %}
<form hx-patch="/update-model-settings" hx-swap="outerHTML" class="grid grid-cols-1 gap-4">
<!-- Query Model -->
<div>
<div class="text-sm opacity-80 mb-1">Query Model</div>
<select name="query_model" class="nb-select w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.query_model==model.id %} selected {% endif %}>{{model.id}}</option>
{% endfor %}
</select>
<p class="text-xs opacity-70 mt-1">Current: <span class="font-mono">{{settings.query_model}}</span></p>
</div>
<!-- Processing Model -->
<div>
<div class="text-sm opacity-80 mb-1">Processing Model</div>
<select name="processing_model" class="nb-select w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.processing_model==model.id %} selected {% endif %}>{{model.id}}</option>
{% endfor %}
</select>
<p class="text-xs opacity-70 mt-1">Current: <span class="font-mono">{{settings.processing_model}}</span></p>
</div>
<!-- Image Processing Model -->
<div>
<div class="text-sm opacity-80 mb-1">Image Processing Model</div>
<select name="image_processing_model" class="nb-select w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.image_processing_model==model.id %} selected {% endif %}>{{model.id}}</option>
{% endfor %}
</select>
<p class="text-xs opacity-70 mt-1">Current: <span class="font-mono">{{settings.image_processing_model}}</span></p>
</div>
<!-- Voice Processing Model -->
<div>
<div class="text-sm opacity-80 mb-1">Voice Processing Model</div>
<select name="voice_processing_model" class="nb-select w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.voice_processing_model==model.id %} selected {% endif %}>{{model.id}}</option>
{% endfor %}
</select>
<p class="text-xs opacity-70 mt-1">Current: <span class="font-mono">{{settings.voice_processing_model}}</span></p>
</div>
<!-- Embedding Model -->
<div>
<div class="text-sm opacity-80 mb-1">Embedding Model</div>
<select name="embedding_model" class="nb-select w-full">
{% for model in available_models.data %}
<option value="{{model.id}}" {% if settings.embedding_model==model.id %} selected {% endif %}>{{model.id}}</option>
{% endfor %}
</select>
<p class="text-xs opacity-70 mt-1">Current: <span class="font-mono">{{settings.embedding_model}} ({{settings.embedding_dimensions}} dims)</span></p>
</div>
<!-- Embedding Dimensions -->
<div>
<div class="text-sm opacity-80 mb-1" for="embedding_dimensions">Embedding Dimensions</div>
<input type="number" id="embedding_dimensions" name="embedding_dimensions" class="nb-input w-full" value="{{ settings.embedding_dimensions }}" required />
</div>
<!-- Alert -->
<div id="embedding-change-alert" class="nb-panel p-3 bg-warning/20 hidden">
<div class="text-sm"><strong>Warning:</strong> Changing dimensions will require re-creating all embeddings. Look up your model's required dimensions or use a model that allows specifying them.</div>
</div>
<div class="flex justify-end">
<button type="submit" class="nb-btn nb-cta btn-sm">Save Model Settings</button>
</div>
</form>
{% endblock %}
Enable Registrations
</label>
<div id="registration-status" class="text-sm mt-2"></div>
</fieldset>
<script>
// Rebind after HTMX swaps
(() => {
const dimensionInput = document.getElementById('embedding_dimensions');
const alertElement = document.getElementById('embedding-change-alert');
const initialDimensions = '{{ settings.embedding_dimensions }}';
if (dimensionInput && alertElement) {
dimensionInput.addEventListener('input', (event) => {
if (String(event.target.value) !== String(initialDimensions)) {
alertElement.classList.remove('hidden');
} else {
alertElement.classList.add('hidden');
}
});
}
})();
</script>
{% endblock %}
</div>
<div class="nb-panel p-4">
<div class="text-sm font-semibold mb-3">Registration</div>
<label class="flex items-center gap-3">
{% block registration_status_input %}
<form hx-patch="/toggle-registrations" hx-swap="outerHTML" hx-trigger="change">
<input name="registration_open" type="checkbox" class="nb-checkbox" {% if settings.registrations_enabled %}checked{% endif %} />
</form>
{% endblock %}
<span class="text-sm">Enable Registrations</span>
</label>
<div id="registration-status" class="text-xs opacity-70 mt-2"></div>
</div>
</section>
</div>
</main>
{% endblock %}
</div>
{% endblock %}

12
html-router/templates/admin/edit_image_prompt_modal.html
View File

@@ -7,17 +7,17 @@ hx-swap="outerHTML"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold mb-4">Edit Image Processing Prompt</h3>
<h3 class="text-xl font-extrabold tracking-tight mb-2">Edit Image Processing Prompt</h3>
<div class="form-control">
<textarea name="image_processing_prompt" class="textarea textarea-bordered h-96 w-full font-mono text-sm">{{
<textarea name="image_processing_prompt" class="nb-input h-96 w-full font-mono text-sm">{{
settings.image_processing_prompt }}</textarea>
<p class="text-xs text-gray-500 mt-1">System prompt used for processing images</p>
<p class="text-xs opacity-70 mt-1">System prompt used for processing images</p>
</div>
{% endblock %}
{% block primary_actions %}
<button type="button" class="btn btn-outline mr-2" id="reset_prompt_button">
<button type="button" class="nb-btn w-full sm:w-auto sm:mr-2" id="reset_prompt_button">
Reset to Default
</button>
@@ -29,10 +29,10 @@ hx-swap="outerHTML"
});
</script>
<button type="submit" class="btn btn-primary">
<button type="submit" class="nb-btn nb-cta w-full sm:w-auto">
<span class="htmx-indicator hidden">
<span class="loading loading-spinner loading-xs mr-2"></span>
</span>
Save Changes
</button>
{% endblock %}
{% endblock %}

12
html-router/templates/admin/edit_ingestion_prompt_modal.html
View File

@@ -7,17 +7,17 @@ hx-swap="outerHTML"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold mb-4">Edit Ingestion Prompt</h3>
<h3 class="text-xl font-extrabold tracking-tight mb-2">Edit Ingestion Prompt</h3>
<div class="form-control">
<textarea name="ingestion_system_prompt" class="textarea textarea-bordered h-96 w-full font-mono text-sm">{{
<textarea name="ingestion_system_prompt" class="nb-input h-96 w-full font-mono text-sm">{{
settings.ingestion_system_prompt }}</textarea>
<p class="text-xs text-gray-500 mt-1">System prompt used for content processing and ingestion</p>
<p class="text-xs opacity-70 mt-1">System prompt used for content processing and ingestion</p>
</div>
{% endblock %}
{% block primary_actions %}
<button type="button" class="btn btn-outline mr-2" id="reset_prompt_button">
<button type="button" class="nb-btn w-full sm:w-auto sm:mr-2" id="reset_prompt_button">
Reset to Default
</button>
@@ -29,10 +29,10 @@ hx-swap="outerHTML"
});
</script>
<button type="submit" class="btn btn-primary">
<button type="submit" class="nb-btn nb-cta w-full sm:w-auto">
<span class="htmx-indicator hidden">
<span class="loading loading-spinner loading-xs mr-2"></span>
</span>
Save Changes
</button>
{% endblock %}
{% endblock %}

12
html-router/templates/admin/edit_query_prompt_modal.html
View File

@@ -7,17 +7,17 @@ hx-swap="outerHTML"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold mb-4">Edit System Prompt</h3>
<h3 class="text-xl font-extrabold tracking-tight mb-2">Edit System Prompt</h3>
<div class="form-control">
<textarea name="query_system_prompt" class="textarea textarea-bordered h-96 w-full font-mono text-sm">{{
<textarea name="query_system_prompt" class="nb-input h-96 w-full font-mono text-sm">{{
settings.query_system_prompt }}</textarea>
<p class="text-xs text-gray-500 mt-1">System prompt used for answering user queries</p>
<p class="text-xs opacity-70 mt-1">System prompt used for answering user queries</p>
</div>
{% endblock %}
{% block primary_actions %}
<button type="button" class="btn btn-outline mr-2" id="reset_prompt_button">
<button type="button" class="nb-btn w-full sm:w-auto sm:mr-2" id="reset_prompt_button">
Reset to Default
</button>
@@ -29,10 +29,10 @@ hx-swap="outerHTML"
});
</script>
<button type="submit" class="btn btn-primary">
<button type="submit" class="nb-btn nb-cta w-full sm:w-auto">
<span class="htmx-indicator hidden">
<span class="loading loading-spinner loading-xs mr-2"></span>
</span>
Save Changes
</button>
{% endblock %}
{% endblock %}

161
html-router/templates/auth/account_settings.html
View File

@@ -3,91 +3,86 @@
{% block title %}Minne - Account{% endblock %}
{% block main %}
<style>
form.htmx-request {
opacity: 0.5;
}
</style>
<main class="container flex-grow flex flex-col mx-auto mt-4 space-y-1">
<h1 class="text-2xl font-bold mb-2">Account Settings</h1>
<div class="form-control">
<label class="label">
<span class="label-text">Email</span>
</label>
<input type="email" name="email" value="{{ user.email }}" class="input text-primary-content input-bordered w-full"
disabled />
</div>
<div class="flex justify-center grow mt-2 sm:mt-4 pb-4">
<div class="container">
<section class="mb-4">
<div class="nb-panel p-3 flex items-center justify-between">
<h1 class="text-xl font-extrabold tracking-tight">Account Settings</h1>
</div>
</section>
<div class="form-control">
<label class="label">
<span class="label-text">API key</span>
</label>
{% block api_key_section %}
{% if user.api_key %}
<div class="relative">
<input id="api_key_input" type="text" name="api_key" value="{{ user.api_key }}"
class="input text-primary-content input-bordered w-full pr-12" disabled />
<button type="button" id="copy_api_key_btn" onclick="copy_api_key()"
class="absolute inset-y-0 cursor-pointer right-0 flex items-center pr-3" title="Copy API key">
{% include "icons/clipboard_icon.html" %}
</button>
</div>
<a href="https://www.icloud.com/shortcuts/66985f7b98a74aaeac6ba29c3f1f0960"
class="btn btn-accent mt-4 w-full">Download iOS shortcut</a>
{% else %}
<button hx-post="/set-api-key" class="btn btn-secondary w-full" hx-swap="outerHTML">
Create API-Key
</button>
{% endif %}
{% endblock %}
</div>
<section class="grid grid-cols-1 lg:grid-cols-2 gap-4 space-y-2">
<!-- Left column -->
<div class="nb-panel p-4 space-y-2 flex flex-col">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Email</div>
<input type="email" name="email" value="{{ user.email }}" class="nb-input w-full" disabled />
</label>
<script>
function copy_api_key() {
const input = document.getElementById('api_key_input');
if (!input) return;
if (navigator.clipboard && window.isSecureContext) {
navigator.clipboard.writeText(input.value)
.then(() => show_toast('API key copied!', 'success'))
.catch(() => show_toast('Copy failed', 'error'));
} else {
show_toast('Copy not supported', 'info');
}
}
</script>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">API Key</div>
{% block api_key_section %}
{% if user.api_key %}
<div class="relative">
<input id="api_key_input" type="text" name="api_key" value="{{ user.api_key }}"
class="nb-input w-full pr-14" disabled />
<button type="button" id="copy_api_key_btn" onclick="copy_api_key()"
class="absolute inset-y-0 right-0 flex items-center px-2 nb-btn btn-sm" aria-label="Copy API key"
title="Copy API key">
{% include "icons/clipboard_icon.html" %}
</button>
</div>
<a href="https://www.icloud.com/shortcuts/66985f7b98a74aaeac6ba29c3f1f0960"
class="nb-btn nb-cta mt-2 w-full">Download iOS shortcut</a>
{% else %}
<button hx-post="/set-api-key" class="nb-btn nb-cta w-full" hx-swap="outerHTML">Create API-Key</button>
{% endif %}
{% endblock %}
</label>
<div class="form-control mt-4">
<label class="label">
<span class="label-text">Timezone</span>
</label>
{% block timezone_section %}
<select name="timezone" class="select w-full" hx-patch="/update-timezone" hx-swap="outerHTML">
{% for tz in timezones %}
<option value="{{ tz }}" {% if tz==user.timezone %}selected{% endif %}>{{ tz }}</option>
{% endfor %}
</select>
{% endblock %}
</div>
<script>
function copy_api_key() {
const input = document.getElementById('api_key_input');
if (!input) return;
if (navigator.clipboard && window.isSecureContext) {
navigator.clipboard.writeText(input.value)
.then(() => show_toast('API key copied!', 'success'))
.catch(() => show_toast('Copy failed', 'error'));
} else {
show_toast('Copy not supported', 'info');
}
}
</script>
<div class="form-control mt-4 hidden">
<button hx-post="/verify-email" class="btn btn-secondary w-full">
Verify Email
</button>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Timezone</div>
{% block timezone_section %}
<select name="timezone" class="nb-select w-full" hx-patch="/update-timezone" hx-swap="outerHTML">
{% for tz in timezones %}
<option value="{{ tz }}" {% if tz==user.timezone %}selected{% endif %}>{{ tz }}</option>
{% endfor %}
</select>
{% endblock %}
</label>
</div>
<!-- Right column -->
<div class="nb-panel p-4 space-y-2">
<div>
{% block change_password_section %}
<button hx-get="/change-password" hx-swap="outerHTML" class="nb-btn w-full">Change Password</button>
{% endblock %}
</div>
<div>
<button hx-delete="/delete-account"
hx-confirm="This action will permanently delete your account and all data associated. Are you sure you want to continue?"
class="nb-btn btn-error w-full">Delete Account</button>
</div>
</div>
</section>
<div id="account-result" class="mt-4"></div>
</div>
<div class="form-control mt-4">
{% block change_password_section %}
<button hx-get="/change-password" hx-swap="outerHTML" class="btn btn-primary w-full">
Change Password
</button>
{% endblock %}
</div>
<div class="form-control mt-4">
<button hx-delete="/delete-account"
hx-confirm="This action will permanently delete your account and all data associated. Are you sure you want to continue?"
class="btn btn-error w-full">
Delete Account
</button>
</div>
<div id="account-result" class="mt-4"></div>
</main>
{% endblock %}
</div>
{% endblock %}

17
html-router/templates/auth/change_password_form.html
View File

@@ -1,5 +1,12 @@
<form hx-patch="/change-password" class="flex flex-col gap-1">
<input name="old_password" class="input w-full" type="password" placeholder="Enter old password"></input>
<input name="new_password" class="input w-full" type="password" placeholder="Enter new password"></input>
<button class="btn btn-primary w-full">Change Password</button>
</form>
<form hx-patch="/change-password" class="flex flex-col gap-3">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Old Password</div>
<input name="old_password" class="nb-input w-full" type="password" placeholder="Enter old password"></input>
</label>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">New Password</div>
<input name="new_password" class="nb-input w-full" type="password" placeholder="Enter new password"></input>
</label>
<button class="nb-btn w-full">Change Password</button>
</form>

83
html-router/templates/auth/signin_form.html
View File

@@ -1,52 +1,43 @@
<style>
form.htmx-request {
opacity: 0.5;
}
</style>
<div class="flex justify-center grow container mx-auto px-4 sm:px-0 sm:max-w-md flex-col">
<h1
class="text-5xl sm:text-6xl py-4 pt-10 font-bold bg-linear-to-r from-primary to-secondary text-center text-transparent bg-clip-text">
Minne
</h1>
<h2 class="text-2xl font-bold text-center mb-8">Login to your account</h2>
<form hx-post="/signin" hx-target="#login-result">
<div class="form-control">
<label class="floating-label">
<span>Email</span>
<input name="email" type="email" placeholder="Email" class="input input-md w-full validator" required />
<div class="validator-hint hidden">Enter valid email address</div>
</label>
<div class="container mx-auto px-4 sm:max-w-md flex-1 flex items-center justify-center">
<div class="w-full nb-card p-5">
<div class="flex items-center justify-between mb-3">
<div class="brand-mark text-3xl font-extrabold tracking-tight">MINNE</div>
<span class="nb-badge">Sign In</span>
</div>
<div class="u-hairline mb-3"></div>
<div class="form-control mt-4">
<label class="floating-label">
<span>Password</span>
<input name="password" type="password" class="input validator w-full" required placeholder="Password"
<form hx-post="/signin" hx-target="#login-result" class="flex flex-col gap-2">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Email</div>
<input name="email" type="email" placeholder="Email" class="nb-input w-full validator" required />
<div class="validator-hint hidden text-xs opacity-70 mt-1">Enter valid email address</div>
</label>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Password</div>
<input name="password" type="password" class="nb-input w-full validator" required placeholder="Password"
minlength="8" />
</div>
<div class="form-control mt-4">
<label class="label cursor-pointer justify-start gap-4">
<input type="checkbox" name="remember_me" class="checkbox " />
<span class="label-text">Remember me</span>
</label>
<div class="mt-1 text-error" id="login-result"></div>
<div class="form-control mt-1">
<label class="label cursor-pointer justify-start gap-3">
<input type="checkbox" name="remember_me" class="nb-checkbox" />
<span class="label-text">Remember me</span>
</label>
</div>
<div class="form-control mt-1">
<button id="submit-btn" class="nb-btn nb-cta w-full">Login</button>
</div>
</form>
<div class="u-hairline my-3"></div>
<div class="text-center text-sm">
Don’t have an account?
<a href="/signup" hx-boost="true" class="nb-link">Sign up</a>
</div>
<div class="mt-4" id="login-result"></div>
<div class="form-control mt-6">
<button id="submit-btn" class="btn btn-primary w-full">
Login
</button>
</div>
</form>
<div class="divider">OR</div>
<div class="text-center text-sm">
Don't have an account?
<a href="/signup" hx-boost="true" class="link link-primary">Sign up</a>
</div>
</div>
</div>

88
html-router/templates/auth/signup_form.html
View File

@@ -3,56 +3,48 @@
{% block title %}Minne - Sign up{% endblock %}
{% block body %}
<style>
form.htmx-request {
opacity: 0.5;
}
</style>
<div class="min-h-[100dvh] flex items-center">
<div class="container mx-auto px-4 sm:max-w-md">
<div class="nb-card p-5">
<div class="flex items-center justify-between mb-3">
<div class="text-3xl font-extrabold tracking-tight">MINNE</div>
<span class="nb-badge">Sign Up</span>
</div>
<div class="u-hairline mb-3"></div>
<div class="min-h-[100dvh] container mx-auto px-4 sm:px-0 sm:max-w-md flex justify-center flex-col">
<h1
class="text-5xl sm:text-6xl py-4 pt-10 font-bold bg-linear-to-r from-primary to-secondary text-center text-transparent bg-clip-text">
Minne
</h1>
<h2 class="text-2xl font-bold text-center mb-8">Create your account</h2>
<form hx-post="/signup" hx-target="#signup-result" class="flex flex-col gap-4">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Email</div>
<input type="email" placeholder="Email" name="email" required class="nb-input w-full validator" />
<div class="validator-hint hidden text-xs opacity-70 mt-1">Enter valid email address</div>
</label>
<form hx-post="/signup" hx-target="#signup-result" class="">
<div class="form-control">
<label class="floating-label">
<span>Email</span>
<input type="email" placeholder="Email" name="email" required class="input input-md w-full validator" />
<div class="validator-hint hidden">Enter valid email address</div>
</label>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Password</div>
<input type="password" name="password" class="nb-input w-full validator" required placeholder="Password"
minlength="8" pattern="(?=.*\d)(?=.*[a-z])(?=.*[A-Z]).{8,}"
title="Must be more than 8 characters, including number, lowercase letter, uppercase letter" />
<p class="validator-hint hidden text-xs opacity-70 mt-1">
Must be more than 8 characters, including
<br />At least one number
<br />At least one lowercase letter
<br />At least one uppercase letter
</p>
</label>
<div class="mt-2 text-error" id="signup-result"></div>
<div class="form-control mt-1">
<button id="submit-btn" class="nb-btn nb-cta w-full">Create Account</button>
</div>
<input type="hidden" name="timezone" id="timezone" />
</form>
<div class="u-hairline my-3"></div>
<div class="text-center text-sm">
Already have an account?
<a href="/signin" hx-boost="true" class="nb-link">Sign in</a>
</div>
</div>
<div class="form-control mt-4">
<label class="floating-label">
<span>Password</span>
<input type="password" name="password" class="input validator w-full" required placeholder="Password"
minlength="8" pattern="(?=.*\d)(?=.*[a-z])(?=.*[A-Z]).{8,}"
title="Must be more than 8 characters, including number, lowercase letter, uppercase letter" />
<p class="validator-hint hidden">
Must be more than 8 characters, including
<br />At least one number
<br />At least one lowercase letter
<br />At least one uppercase letter
</p>
</label>
</div>
<div class="mt-4 text-error" id="signup-result"></div>
<div class="form-control mt-6">
<button id="submit-btn" class="btn btn-primary w-full">
Create Account
</button>
</div>
<input type="hidden" name="timezone" id="timezone" />
</form>
<div class="divider">OR</div>
<div class="text-center text-sm">
Already have an account?
<a href="/signin" hx-boost="true" class="link link-primary">Sign in</a>
</div>
</div>
<script>
@@ -60,4 +52,4 @@
const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone;
document.getElementById("timezone").value = timezone;
</script>
{% endblock %}
{% endblock %}

25
html-router/templates/body_base.html
View File

@@ -2,7 +2,7 @@
{% block body %}
<body class="bg-base-100 relative" hx-ext="head-support">
<body class="relative" hx-ext="head-support">
<div class="drawer lg:drawer-open">
<input id="my-drawer" type="checkbox" class="drawer-toggle" />
<!-- Page Content -->
@@ -10,8 +10,9 @@
<!-- Navbar -->
{% include "navigation_bar.html" %}
<!-- Main Content Area -->
<main class="flex flex-1 overflow-y-auto">
<main class="flex flex-col flex-1 overflow-y-auto">
{% block main %}{% endblock %}
<div class="p32 min-h-[10px]"></div>
</main>
</div>
<!-- Sidebar -->
@@ -21,25 +22,5 @@
</div> <!-- End Drawer -->
<div id="modal"></div>
<div id="toast-container" class="fixed bottom-4 right-4 z-50 space-y-2"></div>
<!-- Add CSS for custom scrollbar -->
<style>
.custom-scrollbar::-webkit-scrollbar {
width: 4px;
}
.custom-scrollbar::-webkit-scrollbar-track {
background: transparent;
}
.custom-scrollbar::-webkit-scrollbar-thumb {
background-color: rgba(0, 0, 0, 0.2);
border-radius: 3px;
}
.custom-scrollbar {
scrollbar-width: thin;
scrollbar-color: rgba(0, 0, 0, 0.2) transparent;
}
</style>
</body>
{% endblock %}

102
html-router/templates/chat/base.html
View File

@@ -9,94 +9,40 @@
{% block main %}
<div class="flex grow relative justify-center mt-2 sm:mt-4">
<div class="container">
<div class="overflow-auto hide-scrollbar">
<section class="mb-3">
<div class="nb-panel p-3 flex items-center justify-between">
<h1 class="text-xl font-extrabold tracking-tight">Chat</h1>
<div class="text-xs opacity-70">Converse with your knowledge</div>
</div>
</section>
<div id="chat-scroll-container" class="overflow-auto hide-scrollbar">
{% include "chat/history.html" %}
{% include "chat/new_message_form.html" %}
</div>
</div>
</div>
<style>
.hide-scrollbar {
-ms-overflow-style: none;
scrollbar-width: none;
}
.hide-scrollbar::-webkit-scrollbar {
display: none;
}
.markdown-content p {
margin-bottom: 0.75em;
}
.markdown-content p:last-child {
margin-bottom: 0;
}
.markdown-content ul,
.markdown-content ol {
margin-top: 0.5em;
margin-bottom: 0.75em;
padding-left: 2em;
}
.markdown-content li {
margin-bottom: 0.25em;
}
.markdown-content pre {
background-color: rgba(0, 0, 0, 0.05);
padding: 0.5em;
border-radius: 4px;
overflow-x: auto;
}
.markdown-content code {
background-color: rgba(0, 0, 0, 0.05);
padding: 0.2em 0.4em;
border-radius: 3px;
font-size: 0.9em;
}
.markdown-content {
line-height: 1.5;
word-wrap: break-word;
}
.markdown-content table {
border-collapse: collapse;
margin: 0.75em 0;
width: 100%;
}
.markdown-content th,
.markdown-content td {
border: 1px solid #ddd;
padding: 6px 12px;
text-align: left;
}
.markdown-content blockquote {
border-left: 4px solid #ddd;
padding-left: 10px;
margin: 0.5em 0 0.5em 0.5em;
color: #666;
}
.markdown-content hr {
border: none;
border-top: 1px solid #ddd;
margin: 0.75em 0;
}
</style>
<script>
function scrollChatToBottom() {
const chatContainer = document.getElementById('chat_container');
if (chatContainer) chatContainer.scrollTop = chatContainer.scrollHeight;
requestAnimationFrame(() => {
const mainScroll = document.querySelector('main');
if (mainScroll) mainScroll.scrollTop = mainScroll.scrollHeight;
const chatScroll = document.getElementById('chat-scroll-container');
if (chatScroll) chatScroll.scrollTop = chatScroll.scrollHeight;
const chatContainer = document.getElementById('chat_container');
if (chatContainer) chatContainer.scrollTop = chatContainer.scrollHeight;
window.scrollTo(0, document.body.scrollHeight);
});
}
window.scrollChatToBottom = scrollChatToBottom;
document.addEventListener('DOMContentLoaded', scrollChatToBottom);
document.body.addEventListener('htmx:afterSwap', scrollChatToBottom);
document.body.addEventListener('htmx:afterSettle', scrollChatToBottom);
</script>
{% endblock %}
{% endblock %}

2
html-router/templates/chat/history.html
View File

@@ -1,4 +1,4 @@
<div id="chat_container" class="pl-3 overflow-y-auto h-[calc(100vh-170px)] sm:h-[calc(100vh-190px)] hide-scrollbar">
<div id="chat_container" class="px-3 pb-44 space-y-3">
{% for message in history %}
{% if message.role == "AI" %}
<div class="chat chat-start">

4
html-router/templates/chat/new_chat_first_response.html
View File

@@ -1,11 +1,9 @@
{% include "chat/streaming_response.html" %}
<!-- OOB swap targeting the form element directly -->
<form id="chat-form" hx-post="/chat/{{conversation.id}}" hx-target="#chat_container" hx-swap="beforeend"
class="relative flex gap-2" hx-swap-oob="true">
<textarea autofocus required name="content" placeholder="Type your message..." rows="2"
class="textarea textarea-ghost rounded-2xl rounded-b-none h-24 sm:rounded-b-2xl pr-8 bg-base-200 flex-grow resize-none"
id="chat-input"></textarea>
class="nb-input h-24 pr-8 pl-2 pt-2 pb-2 flex-grow resize-none" id="chat-input"></textarea>
<button type="submit" class="absolute p-2 cursor-pointer right-0.5 btn-ghost btn-sm top-1">
{% include "icons/send_icon.html" %}
</button>

23
html-router/templates/chat/new_message_form.html
View File

@@ -1,13 +1,14 @@
<div class="absolute w-full mx-auto max-w-3xl p-0 pb-0 sm:pb-4 left-0 right-0 bottom-0 z-10">
<form hx-post="{% if conversation %} /chat/{{conversation.id}} {% else %} /chat {% endif %}"
hx-target="#chat_container" hx-swap="beforeend" class="relative flex gap-2" id="chat-form">
<textarea autofocus required name="content" placeholder="Type your message..." rows="2"
class="textarea textarea-ghost rounded-2xl rounded-b-none h-24 sm:rounded-b-2xl pr-8 bg-base-200 flex-grow resize-none focus:outline-none focus:bg-base-200"
id="chat-input"></textarea>
<button type="submit" class="absolute p-2 cursor-pointer right-0.5 btn-ghost btn-sm top-6">{% include
"icons/send_icon.html" %}
</button>
</form>
<div class="fixed bottom-0 left-0 right-0 lg:left-72 z-20">
<div class="mx-auto max-w-3xl px-4 pb-3">
<div class="nb-panel p-2">
<form hx-post="{% if conversation %} /chat/{{conversation.id}} {% else %} /chat {% endif %}"
hx-target="#chat_container" hx-swap="beforeend" class="relative flex gap-2 items-end" id="chat-form">
<textarea autofocus required name="content" placeholder="Type your message…" rows="3"
class="nb-input flex-grow min-h-24 pr-10 pl-2 pt-2 pb-2 resize-none" id="chat-input"></textarea>
<button type="submit" class="nb-btn nb-cta h-10 px-3">{% include "icons/send_icon.html" %}</button>
</form>
</div>
</div>
</div>
<script>
@@ -23,4 +24,4 @@
document.getElementById('chat-input').value = ''; // Clear the textarea
}
});
</script>
</script>

20
html-router/templates/chat/reference_list.html
View File

@@ -1,8 +1,8 @@
<div class="relative my-2">
<button id="references-toggle-{{message.id}}"
class="text-xs text-blue-500 hover:text-blue-700 hover:underline focus:outline-none flex items-center"
class="nb-btn btn-xs bg-base-100 hover:bg-base-200 flex items-center"
onclick="toggleReferences('{{message.id}}')">
References
REFERENCES
{% include "icons/chevron_icon.html" %}
</button>
<div id="references-content-{{message.id}}" class="hidden max-w-full mt-1">
@@ -10,7 +10,7 @@
{% for reference in message.references %}
<div class="reference-badge-container" data-reference="{{reference}}" data-message-id="{{message.id}}"
data-index="{{loop.index}}">
<span class="badge badge-xs badge-neutral truncate max-w-[20ch] overflow-hidden text-left block cursor-pointer">
<span class="nb-badge truncate max-w-[20ch] overflow-hidden text-left block cursor-pointer">
{{reference}}
</span>
</div>
@@ -80,7 +80,7 @@
function createTooltip() {
const tooltip = document.createElement('div');
tooltip.id = tooltipId;
tooltip.className = 'fixed z-[9999] bg-neutral-800 text-white p-3 rounded-md shadow-lg text-sm w-72 max-w-xs border border-neutral-700 hidden';
tooltip.className = 'reference-tooltip hidden';
tooltip.innerHTML = '<div class="animate-pulse">Loading...</div>';
document.body.appendChild(tooltip);
return tooltip;
@@ -135,15 +135,3 @@
});
}
</script>
<style>
#references-toggle- {
{
message.id
}
}
svg {
transition: transform 0.2s ease;
}
</style>

6
html-router/templates/chat/streaming_response.html
View File

@@ -25,7 +25,7 @@
e.preventDefault();
window.markdownBuffer[msgId] = (window.markdownBuffer[msgId] || '') + (e.detail.data || '');
el.innerHTML = marked.parse(window.markdownBuffer[msgId].replace(/\\n/g, '\n'));
if (typeof scrollChatToBottom === "function") scrollChatToBottom();
if (typeof window.scrollChatToBottom === "function") window.scrollChatToBottom();
});
document.body.addEventListener('htmx:sseClose', function () {
const msgId = '{{ user_message.id }}';
@@ -33,7 +33,7 @@
if (el && window.markdownBuffer[msgId]) {
el.innerHTML = marked.parse(window.markdownBuffer[msgId].replace(/\\n/g, '\n'));
delete window.markdownBuffer[msgId];
if (typeof scrollChatToBottom === "function") scrollChatToBottom();
if (typeof window.scrollChatToBottom === "function") window.scrollChatToBottom();
}
});
</script>
</script>

19
html-router/templates/content/base.html
View File

@@ -3,14 +3,15 @@
{% block title %}Minne - Content{% endblock %}
{% block main %}
<main id="main_section" class="flex justify-center grow mt-2 sm:mt-4 gap-6 mb-10">
<main id="main_section" class="flex justify-center grow mt-2 sm:mt-4 gap-6 mb-10 w-full">
<div class="container">
<div class="flex items-center justify-between mb-4">
<h2 class="text-2xl font-bold">Content</h2>
<div class="nb-panel p-3 mb-4 flex items-center justify-between">
<h2 class="text-xl font-extrabold tracking-tight">Content</h2>
<form hx-get="/content" hx-target="#main_section" hx-swap="outerHTML" hx-push-url="true"
class="flex items-center gap-2 mt-2 sm:mt-0">
<div class="form-control">
<select name="category" class="select select-bordered">
<input type="hidden" name="page" value="1" />
<div>
<select name="category" class="nb-select">
<option value="">All Categories</option>
{% for category in categories %}
<option value="{{ category }}" {% if selected_category==category %}selected{% endif %}>{{ category }}
@@ -18,13 +19,11 @@
{% endfor %}
</select>
</div>
<button type="submit" class="btn btn-primary btn-sm">Filter</button>
<button type="submit" class="nb-btn btn-sm">Filter</button>
</form>
</div>
<div id="text_content_cards">
{% include "content/content_list.html" %}
</div>
{% include "content/content_list.html" %}
</div>
</main>
{% endblock %}
{% endblock %}

146
html-router/templates/content/content_list.html
View File

@@ -1,58 +1,102 @@
<div class="columns-1 md:columns-2 2xl:columns-3 gap-4" id="text_content_cards">
{% for text_content in text_contents %}
<div class="card cursor-pointer mb-4 bg-base-100 shadow break-inside-avoid-column"
hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML">
{% if text_content.url_info %}
<figure>
<img src="/file/{{text_content.url_info.image_id}}" alt="website screenshot" />
</figure>
{% endif %}
{% if text_content.file_info.mime_type == "image/png" or text_content.file_info.mime_type == "image/jpeg" %}
<figure>
<img src="/file/{{text_content.file_info.id}}" alt="{{text_content.file_info.file_name}}" />
</figure>
{% endif %}
<div class="card-body max-w-[95vw]">
<h2 class="card-title truncate">
{% if text_content.url_info %}
{{text_content.url_info.title}}
{% elif text_content.file_info %}
{{text_content.file_info.file_name}}
{% else %}
{{text_content.text}}
{% endif %}
</h2>
<div class="flex items-center justify-between">
<p class="text-xs opacity-60">
{{ text_content.created_at | datetimeformat(format="short", tz=user.timezone) }}
</p>
<div class="badge badge-soft badge-secondary mr-2">{{ text_content.category }}</div>
<div class="flex gap-2" hx-on:click="event.stopPropagation()">
{% set has_pagination = pagination is defined %}
{% set query_suffix = '' %}
{% if page_query is defined and page_query %}
{% set query_suffix = page_query %}
{% endif %}
<div id="text_content_cards" class="space-y-6">
{% if text_contents|length > 0 %}
<div class="nb-masonry w-full">
{% for text_content in text_contents %}
<article class="nb-card cursor-pointer mx-auto mb-4 w-full max-w-[92vw] space-y-3 sm:max-w-none"
hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML">
{% if text_content.url_info %}
<figure class="-mx-4 -mt-4 border-b-2 border-neutral bg-base-200">
<img class="w-full h-auto" src="/file/{{text_content.url_info.image_id}}" alt="website screenshot" />
</figure>
{% endif %}
{% if text_content.file_info.mime_type == "image/png" or text_content.file_info.mime_type == "image/jpeg" %}
<figure class="-mx-4 -mt-4 border-b-2 border-neutral bg-base-200">
<img class="w-full h-auto" src="/file/{{text_content.file_info.id}}" alt="{{text_content.file_info.file_name}}" />
</figure>
{% endif %}
<div class="space-y-3 break-words">
<h2 class="text-lg font-extrabold tracking-tight truncate">
{% if text_content.url_info %}
<button class="btn-btn-square btn-ghost btn-sm">
<a href="{{text_content.url_info.url}}" target="_blank" rel="noopener noreferrer">
{{text_content.url_info.title}}
{% elif text_content.file_info %}
{{text_content.file_info.file_name}}
{% else %}
{{text_content.text}}
{% endif %}
</h2>
<div class="flex flex-wrap items-center justify-between gap-3">
<p class="text-xs opacity-60 shrink-0">
{{ text_content.created_at | datetimeformat(format="short", tz=user.timezone) }}
</p>
<span class="nb-badge">{{ text_content.category }}</span>
<div class="flex gap-2" hx-on:click="event.stopPropagation()">
{% if text_content.url_info %}
<a href="{{text_content.url_info.url}}" target="_blank" rel="noopener noreferrer"
class="nb-btn btn-square btn-sm" aria-label="Open source link">
{% include "icons/link_icon.html" %}
</a>
</button>
{% endif %}
<button hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/read_icon.html" %}
</button>
<button hx-get="/content/{{ text_content.id }}" hx-target="#modal" hx-swap="innerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/edit_icon.html" %}
</button>
<button hx-delete="/content/{{ text_content.id }}" hx-target="#text_content_cards" hx-swap="outerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/delete_icon.html" %}
</button>
{% endif %}
<button hx-get="/content/{{ text_content.id }}/read" hx-target="#modal" hx-swap="innerHTML"
class="nb-btn btn-square btn-sm" aria-label="Read content">
{% include "icons/read_icon.html" %}
</button>
<button hx-get="/content/{{ text_content.id }}" hx-target="#modal" hx-swap="innerHTML"
class="nb-btn btn-square btn-sm" aria-label="Edit content">
{% include "icons/edit_icon.html" %}
</button>
<button hx-delete="/content/{{ text_content.id }}" hx-target="#text_content_cards" hx-swap="outerHTML"
class="nb-btn btn-square btn-sm" aria-label="Delete content">
{% include "icons/delete_icon.html" %}
</button>
</div>
</div>
<p class="text-sm leading-relaxed">
{{ text_content.instructions }}
</p>
</div>
<p class="mt-2">
{{ text_content.instructions }}
</p>
</article>
{% endfor %}
</div>
{% else %}
<div class="nb-card p-8 text-center text-sm opacity-70">
No content found.
</div>
{% endif %}
{% if has_pagination and pagination.total_items > 0 %}
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
<span class="text-sm opacity-70">
Showing {{ pagination.start_index }}-{{ pagination.end_index }} of {{ pagination.total_items }} items
</span>
<div class="flex gap-2">
{% set prev_enabled = pagination.previous_page is not none %}
<button type="button" class="nb-btn btn-outline btn-sm"
{% if prev_enabled %}
hx-get="/content?page={{ pagination.previous_page }}{{ query_suffix }}"
{% else %}
disabled
{% endif %}
hx-target="#main_section" hx-swap="outerHTML" hx-push-url="true">
Previous
</button>
{% set next_enabled = pagination.next_page is not none %}
<button type="button" class="nb-btn btn-outline btn-sm"
{% if next_enabled %}
hx-get="/content?page={{ pagination.next_page }}{{ query_suffix }}"
{% else %}
disabled
{% endif %}
hx-target="#main_section" hx-swap="outerHTML" hx-push-url="true">
Next
</button>
</div>
</div>
{% endfor %}
</div>
{% endif %}
</div>

61
html-router/templates/content/edit_text_content_modal.html
View File

@@ -8,34 +8,49 @@ flex flex-col min-h-[95%] w-11/12 max-w-[90ch] max-h-[95%]
hx-patch="/content/{{text_content.id}}"
hx-target="#main_section"
hx-swap="outerHTML"
class="flex flex-col flex-1 h-full"
class="flex flex-col flex-1 h-full min-h-0"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold">Edit Content</h3>
<div class="form-control">
<label class="floating-label">
<span class="label-text">Context</span>
<input type="text" name="context" value="{{ text_content.context }}" class="w-full input input-bordered">
</label>
</div>
<div class="form-control">
<label class="floating-label">
<span class="label-text">Category</span>
<input type="text" name="category" value="{{ text_content.category }}" class="w-full input input-bordered">
</label>
</div>
<div class="form-control flex-1 flex flex-col min-h-0">
<label class="floating-label flex-1 flex flex-col min-h-0">
<span class="label-text">Text</span>
<textarea name="text" class="textarea textarea-bordered w-full flex-1 min-h-[200px] h-full resize-none">{{
text_content.text }}</textarea>
<h3 class="text-xl font-extrabold tracking-tight">Edit Content</h3>
<div class="flex flex-col gap-3 flex-1 min-h-0">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Context</div>
<input type="text" name="context" value="{{ text_content.context }}" class="nb-input w-full">
</label>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Category</div>
<input type="text" name="category" value="{{ text_content.category }}" class="nb-input w-full">
</label>
<label class="w-full flex-1 flex flex-col min-h-0">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Text</div>
<textarea name="text" class="nb-input w-full flex-1 min-h-0 h-full resize-none overflow-y-auto">{{ text_content.text
}}</textarea>
</label>
</div>
<script>
(function () {
const form = document.getElementById('modal_form');
if (!form) return;
if (document.getElementById('main_section')) {
form.setAttribute('hx-target', '#main_section');
form.setAttribute('hx-swap', 'outerHTML');
return;
}
if (document.getElementById('latest_content_section')) {
form.setAttribute('hx-target', '#latest_content_section');
form.setAttribute('hx-swap', 'outerHTML');
return;
}
form.removeAttribute('hx-target');
form.setAttribute('hx-swap', 'none');
})();
</script>
{% endblock %}
{% block primary_actions %}
<button type="submit" class="btn btn-primary">
Save Changes
</button>
{% endblock %}
<button type="submit" class="nb-btn nb-cta">Save Changes</button>
{% endblock %}

6
html-router/templates/content/read_content_modal.html
View File

@@ -4,14 +4,14 @@
{% block modal_content %}
{% if text_content.url_info.image_id %}
<img class="rounded-t-md overflow-clip" src="/file/{{text_content.url_info.image_id}}" alt="Screenshot of the site" />
<img class="w-full border-b-2 border-neutral" src="/file/{{text_content.url_info.image_id}}" alt="Screenshot of the site" />
{% endif %}
{% if text_content.file_info.mime_type == "image/png" or text_content.file_info.mime_type == "image/jpeg" %}
<figure>
<img src="/file/{{text_content.file_info.id}}" alt="{{text_content.file_info.file_name}}" />
</figure>
{% endif %}
<div id="reader-{{text_content.id}}" class="markdown-content prose" data-content="{{text_content.text | escape }}">
<div id="reader-{{text_content.id}}" class="markdown-content prose-tufte" data-content="{{text_content.text | escape }}">
{{text_content.text | escape }}
</div>
@@ -39,4 +39,4 @@
{% endblock %}
{% block primary_actions %}
{% endblock %}
{% endblock %}

102
html-router/templates/dashboard/active_jobs.html
View File

@@ -1,49 +1,63 @@
{% block active_jobs_section %}
<ul id="active_jobs_section" class="list">
<div class="flex items-center gap-4">
<li class="py-4 text-2xl font-bold tracking-wide">Active Tasks</li>
<button class="cursor-pointer scale-75" hx-get="/active-jobs" hx-target="#active_jobs_section" hx-swap="outerHTML">
<section id="active_jobs_section" class="nb-panel p-4 space-y-4 mt-6 sm:mt-8">
<header class="flex flex-wrap items-center justify-between gap-3">
<h2 class="text-xl font-extrabold tracking-tight">Active Tasks</h2>
<button class="nb-btn btn-square btn-sm" hx-get="/active-jobs" hx-target="#active_jobs_section" hx-swap="outerHTML"
aria-label="Refresh active tasks">
{% include "icons/refresh_icon.html" %}
</button>
</div>
{% for item in active_jobs %}
<li class="list-row">
<div class="bg-secondary rounded-box size-10 flex justify-center items-center text-secondary-content">
{% if item.content.Url %}
{% include "icons/link_icon.html" %}
{% elif item.content.File %}
{% include "icons/document_icon.html" %}
{% else %}
{% include "icons/bars_icon.html" %}
{% endif %}
</div>
<div>
<div class="[&:before]:content-['Status:_'] [&:before]:opacity-60">
{% if item.status.name == "InProgress" %}
In Progress, attempt {{item.status.attempts}}
{% elif item.status.name == "Error" %}
Error: {{item.status.message}}
{% else %}
{{item.status.name}}
{% endif %}
</header>
{% if active_jobs %}
<ul class="flex flex-col gap-3 list-none p-0 m-0">
{% for item in active_jobs %}
<li class="nb-panel p-3 flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
<div class="flex items-center gap-3 flex-1 min-w-0">
<div class="size-10 shrink-0 flex items-center justify-center border-2 border-neutral bg-transparent">
{% if item.content.Url %}
{% include "icons/link_icon.html" %}
{% elif item.content.File %}
{% include "icons/document_icon.html" %}
{% else %}
{% include "icons/bars_icon.html" %}
{% endif %}
</div>
<div class="space-y-1">
<div class="text-sm font-semibold">
{% if item.status.name == "InProgress" %}
In progress, attempt {{ item.status.attempts }}
{% elif item.status.name == "Error" %}
Error: {{ item.status.message }}
{% else %}
{{ item.status.name }}
{% endif %}
</div>
<div class="text-xs font-semibold opacity-60">
{{ item.created_at|datetimeformat(format="short", tz=user.timezone) }}
</div>
</div>
</div>
<div class="text-xs font-semibold opacity-60">
{{item.created_at|datetimeformat(format="short", tz=user.timezone)}} </div>
</div>
<p class="list-col-wrap text-xs [&:before]:content-['Content:_'] [&:before]:opacity-60">
{% if item.content.Url %}
{{item.content.Url.url}}
{% elif item.content.File %}
{{item.content.File.file_info.file_name}}
{% else %}
{{item.content.Text.text}}
{% endif %}
</p>
<button hx-delete="/jobs/{{item.id}}" hx-target="#active_jobs_section" hx-swap="outerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/delete_icon.html" %}
</button>
</li>
{% endfor %}
</ul>
<div class="sm:flex-1 sm:text-right">
<p class="text-xs opacity-80 leading-snug break-words">
{% if item.content.Url %}
{{ item.content.Url.url }}
{% elif item.content.File %}
{{ item.content.File.file_info.file_name }}
{% else %}
{{ item.content.Text.text }}
{% endif %}
</p>
</div>
<div class="flex items-center justify-end gap-2">
<button hx-delete="/jobs/{{ item.id }}" hx-target="#active_jobs_section" hx-swap="outerHTML"
class="nb-btn btn-square btn-sm" aria-label="Cancel task">
{% include "icons/delete_icon.html" %}
</button>
</div>
</li>
{% endfor %}
</ul>
{% endif %}
</section>
{% endblock %}

14
html-router/templates/dashboard/base.html
View File

@@ -7,8 +7,20 @@
{% endblock %}
{% block main %}
<div class="flex justify-center grow mt-2 sm:mt-4 pb-4">
<div class="flex justify-center grow mt-2 sm:mt-4 pb-4 w-full">
<div class="container">
<section class="mb-4">
<div class="nb-panel p-3 flex items-center justify-between">
<h1 class="text-xl font-extrabold tracking-tight">Dashboard</h1>
<button class="nb-btn nb-cta" hx-get="/ingress-form" hx-target="#modal" hx-swap="innerHTML">
{% include "icons/send_icon.html" %}
<span class="ml-2">Add Content</span>
</button>
</div>
</section>
{% include "dashboard/statistics.html" %}
{% include "dashboard/recent_content.html" %}
{% include "dashboard/active_jobs.html" %}

64
html-router/templates/dashboard/current_task.html
View File

@@ -1,32 +1,40 @@
{% for task in tasks %}
<li class="list-row" hx-ext="sse" sse-connect="/task/status-stream?task_id={{task.id}}" sse-close="close_stream">
<div class="bg-secondary rounded-box size-10 flex justify-center items-center text-secondary-content"
sse-swap="stop_loading" hx-swap="innerHTML">
<span class="loading loading-spinner loading-xl"></span>
</div>
<div>
<div class="flex gap-1">
<div sse-swap="status" hx-swap="innerHTML">
Created
</div>
<div hx-get="/content/recent" hx-target="#latest_content_section" hx-swap="outerHTML"
hx-trigger="sse:update_latest_content"></div>
<li class="nb-panel p-3 flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between"
hx-ext="sse" sse-connect="/task/status-stream?task_id={{task.id}}" sse-close="close_stream">
<div class="flex items-center gap-3 flex-1 min-w-0">
<div class="size-10 flex items-center justify-center border-2 border-neutral bg-transparent"
sse-swap="stop_loading" hx-swap="innerHTML">
<span class="loading loading-spinner loading-md"></span>
</div>
<div class="space-y-1">
<div class="text-sm font-semibold flex gap-2 items-center">
<span sse-swap="status" hx-swap="innerHTML">Created</span>
<div hx-get="/content/recent" hx-target="#latest_content_section" hx-swap="outerHTML"
hx-trigger="sse:update_latest_content"></div>
</div>
<div class="text-xs font-semibold opacity-60">
{{task.created_at|datetimeformat(format="short", tz=user.timezone)}}
</div>
</div>
<div class="text-xs font-semibold opacity-60">
{{task.created_at|datetimeformat(format="short", tz=user.timezone)}} </div>
</div>
<p class="list-col-wrap text-xs [&:before]:content-['Content:_'] [&:before]:opacity-60">
{% if task.content.Url %}
{{task.content.Url.url}}
{% elif task.content.File %}
{{task.content.File.file_info.file_name}}
{% else %}
{{task.content.Text.text}}
{% endif %}
</p>
<button hx-delete="/jobs/{{task.id}}" hx-target="#active_jobs_section" hx-swap="outerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/delete_icon.html" %}
</button>
<div class="sm:flex-1 sm:text-right">
<p class="text-xs opacity-80 leading-snug break-words">
{% if task.content.Url %}
{{task.content.Url.url}}
{% elif task.content.File %}
{{task.content.File.file_info.file_name}}
{% else %}
{{task.content.Text.text}}
{% endif %}
</p>
</div>
<div class="flex items-center justify-end gap-2">
<button hx-delete="/jobs/{{task.id}}" hx-target="#active_jobs_section" hx-swap="outerHTML"
class="nb-btn btn-square btn-sm" aria-label="Cancel task">
{% include "icons/delete_icon.html" %}
</button>
</div>
</li>
{% endfor %}
{% endfor %}

25
html-router/templates/dashboard/statistics.html Normal file
View File

@@ -0,0 +1,25 @@
<section class="mb-4 sm:mt-4">
<h2 class="text-2xl font-extrabold tracking-tight mb-3">Overview</h2>
<div class="grid grid-cols-1 sm:grid-cols-2 xl:grid-cols-4 gap-4">
<div class="nb-stat">
<div class="text-xs opacity-70">Total Documents</div>
<div class="text-3xl font-extrabold">{{ stats.total_documents }}</div>
<div class="text-xs opacity-60">+{{ stats.new_documents_week }} this week</div>
</div>
<div class="nb-stat">
<div class="text-xs opacity-70">Text Chunks</div>
<div class="text-3xl font-extrabold">{{ stats.total_text_chunks }}</div>
<div class="text-xs opacity-60">+{{ stats.new_text_chunks_week }} this week</div>
</div>
<div class="nb-stat">
<div class="text-xs opacity-70">Knowledge Entities</div>
<div class="text-3xl font-extrabold">{{ stats.total_entities }}</div>
<div class="text-xs opacity-60">+{{ stats.new_entities_week }} this week</div>
</div>
<div class="nb-stat">
<div class="text-xs opacity-70">Conversations</div>
<div class="text-3xl font-extrabold">{{ stats.total_conversations }}</div>
<div class="text-xs opacity-60">+{{ stats.new_conversations_week }} this week</div>
</div>
</div>
</section>

3
html-router/templates/head_base.html
View File

@@ -19,6 +19,7 @@
<script src="/assets/toast.js" defer></script>
<script src="/assets/htmx-head-ext.js" defer></script>
<script src="/assets/marked.min.js" defer></script>
<script src="/assets/knowledge-graph.js" defer></script>
<!-- Icons -->
<link rel="icon" href="/assets/icon/favicon.ico">
@@ -76,4 +77,4 @@
window.renderAllMarkdown = renderAllMarkdown;
</script>
</html>
</html>

52
html-router/templates/ingestion_modal.html
View File

@@ -4,39 +4,37 @@ hx-post="/ingress-form"
enctype="multipart/form-data"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold">Add new content</h3>
<div class="form-control">
<label class="floating-label">
<span>Content</span>
<textarea name="content" class="textarea input-bordered w-full"
placeholder="Enter the content you want to ingest, it can be an URL or a text snippet">{{ content }}</textarea>
<h3 class="text-xl font-extrabold tracking-tight">Add New Content</h3>
<div class="flex flex-col gap-3">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Content</div>
<textarea name="content" class="nb-input w-full min-h-28"
placeholder="Paste a URL or type/paste text to ingest…">{{ content }}</textarea>
</label>
</div>
<div class="form-control">
<label class="floating-label">
<span>Context</span>
<textarea name="context" class="textarea w-full"
placeholder="Enter context for the AI here, help it understand what its seeing or how it should relate to the database">{{
context }}</textarea>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Context</div>
<textarea name="context" class="nb-input w-full min-h-24"
placeholder="Optional: add context to guide how the content should be interpreted…">{{ context }}</textarea>
</label>
</div>
<div class="form-control">
<label class="floating-label">
<span>Category</span>
<input type="text" name="category" class="input input-bordered validator w-full" value="{{ category }}"
list="category-list" required />
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Category</div>
<input type="text" name="category" class="nb-input validator w-full" value="{{ category }}" list="category-list" required />
<datalist id="category-list">
{% for category in user_categories %}
<option value="{{ category }}" />
{% endfor %}
</datalist>
<div class="validator-hint hidden">Category is required</div>
<div class="validator-hint hidden text-xs opacity-70 mt-1">Category is required</div>
</label>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Files</div>
<input type="file" name="files" multiple class="file-input w-full rounded-none border-2 border-neutral" />
</label>
</div>
<div class="form-control">
<label class="label label-text">Files</label>
<input type="file" name="files" multiple class="file-input file-input-bordered w-full" />
</div>
<div id="error-message" class="text-error text-center {% if not error %}hidden{% endif %}">{{ error }}</div>
<script>
(function () {
@@ -54,7 +52,7 @@ enctype="multipart/form-data"
</script>
{% endblock %}
{% block primary_actions %}
<button type="submit" class="btn btn-primary">
Save Changes
<button type="submit" class="nb-btn nb-cta">
Add Content
</button>
{% endblock %}
{% endblock %}

35
html-router/templates/knowledge/base.html
View File

@@ -3,22 +3,23 @@
{% block title %}Minne - Knowledge{% endblock %}
{% block main %}
<main id="knowledge_pane" class="flex justify-center grow mt-2 sm:mt-4 gap-6 mb-10">
<div id="knowledge_pane" class="flex justify-center grow mt-2 sm:mt-4 gap-6">
<div class="container">
<div class="flex flex-col sm:flex-row justify-between items-start sm:items-center mb-4">
<h2 class="text-2xl font-bold">Entities</h2>
<div class="nb-panel p-3 mb-4 flex flex-col sm:flex-row justify-between items-start sm:items-center">
<h2 class="text-xl font-extrabold tracking-tight">Knowledge Entities</h2>
<form hx-get="/knowledge" hx-target="#knowledge_pane" hx-push-url="true" hx-swap="outerHTML"
class="flex items-center gap-4 mt-2 sm:mt-0">
<div class="form-control">
<select name="entity_type" class="select select-bordered">
class="flex items-center gap-2 mt-2 sm:mt-0">
<input type="hidden" name="page" value="1" />
<div>
<select name="entity_type" class="nb-select">
<option value="">All Types</option>
{% for type in entity_types %}
<option value="{{ type }}" {% if selected_entity_type==type %}selected{% endif %}>{{ type }}</option>
{% endfor %}
</select>
</div>
<div class="form-control">
<select name="content_category" class="select select-bordered">
<div>
<select name="content_category" class="nb-select">
<option value="">All Categories</option>
{% for category in content_categories %}
<option value="{{ category }}" {% if selected_content_category==category %}selected{% endif %}>{{ category
@@ -26,16 +27,20 @@
{% endfor %}
</select>
</div>
<button type="submit" class="btn btn-primary btn-sm">Filter</button>
<button type="submit" class="nb-btn btn-sm">Filter</button>
</form>
</div>
{% include "knowledge/entity_list.html" %}
<h2 class="text-2xl font-bold mb-2 mt-10">Relationships</h2>
{% include "knowledge/relationship_table.html" %}
<div class="rounded-box overflow-clip mt-10 shadow">
{{ plot_html | safe }}
<h2 class="text-2xl font-bold mb-2 mt-10 ">Graph</h2>
<div class="nb-card mt-4 p-2">
<div id="knowledge-graph" class="w-full" style="height: 640px;"
data-entity-type="{{ selected_entity_type | default(value='') }}"
data-content-category="{{ selected_content_category | default(value='') }}">
</div>
</div>
{% include "knowledge/entity_list.html" %}
<h2 class="text-2xl font-bold mb-2 mt-2">Relationships</h2>
{% include "knowledge/relationship_table.html" %}
</div>
</main>
</div>
{% endblock %}

24
html-router/templates/knowledge/edit_knowledge_entity_modal.html
View File

@@ -7,18 +7,18 @@ hx-swap="outerHTML"
{% endblock %}
{% block modal_content %}
<h3 class="text-lg font-bold">Edit Entity</h3>
<h3 class="text-xl font-extrabold tracking-tight">Edit Entity</h3>
<div class="form-control">
<label class="floating-label">
<span class="label-text">Name</span>
<input type="text" name="name" value="{{ entity.name }}" class="input input-bordered w-full">
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Name</div>
<input type="text" name="name" value="{{ entity.name }}" class="nb-input w-full">
</label>
</div>
<div class="form-control relative" style="margin-top: -1.5rem;">
<div class="absolute !left-3 !top-2.5 z-50 p-0.5 bg-white text-xs text-light">Type</div>
    <select name="entity_type" class="select w-full">
<div class="absolute !left-3 !top-2.5 z-50 p-0.5 bg-base-100 text-xs">Type</div>
    <select name="entity_type" class="nb-select w-full">
<option disabled>You must select a type</option>
     {% for et in entity_types %}
      <option value="{{ et }}" {% if entity.entity_type==et %}selected{% endif %}>{{ et }}</option>
@@ -29,15 +29,13 @@ hx-swap="outerHTML"
<input type="text" name="id" value="{{ entity.id }}" class="hidden">
<div class="form-control">
<label class="floating-label">
<span class="label-text">Description</span>
<textarea name="description" class="w-full textarea textarea-bordered h-32">{{ entity.description }}</textarea>
<label class="w-full">
<div class="text-xs uppercase tracking-wide opacity-70 mb-1">Description</div>
<textarea name="description" class="nb-input w-full h-32">{{ entity.description }}</textarea>
</label>
</div>
{% endblock %}
{% block primary_actions %}
<button type="submit" class="btn btn-primary">
Save Changes
</button>
{% endblock %}
<button type="submit" class="nb-btn nb-cta">Save Changes</button>
{% endblock %}

76
html-router/templates/knowledge/entity_list.html
View File

@@ -1,25 +1,61 @@
<div class="grid md:grid-cols-2 2xl:grid-cols-3 gap-4" id="entity-list">
{% for entity in entities %}
<div class="card min-w-72 bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">{{entity.name}}
<span class="badge badge-xs badge-primary">{{entity.entity_type}}</span>
</h2>
<div class="flex justify-between items-center">
<p class="text-xs opacity-60">{{entity.updated_at | datetimeformat(format="short", tz=user.timezone)}}</p>
<div>
<button hx-get="/knowledge-entity/{{entity.id}}" hx-target="#modal" hx-swap="innerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/edit_icon.html" %}
</button>
<button hx-delete="/knowledge-entity/{{entity.id}}" hx-target="#entity-list" hx-swap="outerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/delete_icon.html" %}
</button>
{% set query_suffix = '' %}
{% if page_query is defined and page_query %}
{% set query_suffix = page_query %}
{% endif %}
<div id="entity-list" class="space-y-6 mt-6">
{% if visible_entities|length > 0 %}
<div class="grid md:grid-cols-2 2xl:grid-cols-3 gap-4">
{% for entity in visible_entities %}
<div class="card min-w-72 bg-base-100 shadow">
<div class="card-body">
<h2 class="card-title">{{entity.name}}
<span class="badge badge-xs badge-primary">{{entity.entity_type}}</span>
</h2>
<div class="flex justify-between items-center">
<p class="text-xs opacity-60">{{entity.updated_at | datetimeformat(format="short", tz=user.timezone)}}</p>
<div>
<button hx-get="/knowledge-entity/{{entity.id}}" hx-target="#modal" hx-swap="innerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/edit_icon.html" %}
</button>
<button hx-delete="/knowledge-entity/{{entity.id}}" hx-target="#entity-list" hx-swap="outerHTML"
class="btn btn-square btn-ghost btn-sm">
{% include "icons/delete_icon.html" %}
</button>
</div>
</div>
<p>{{entity.description}}</p>
</div>
<p>{{entity.description}}</p>
</div>
{% endfor %}
</div>
{% else %}
<div class="nb-card p-8 text-center text-sm opacity-70">
No knowledge entities found.
</div>
{% endif %}
{% if pagination.total_items > 0 %}
<div class="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between mt-2">
<span class="text-sm opacity-70">
Showing {{ pagination.start_index }}-{{ pagination.end_index }} of {{ pagination.total_items }} entities
</span>
<div class="flex gap-2">
{% set prev_enabled = pagination.previous_page is not none %}
<button type="button" class="nb-btn btn-outline btn-sm" {% if prev_enabled %}
hx-get="/knowledge?page={{ pagination.previous_page }}{{ query_suffix }}" {% else %} disabled {% endif %}
hx-target="#knowledge_pane" hx-swap="outerHTML" hx-push-url="true">
Previous
</button>
{% set next_enabled = pagination.next_page is not none %}
<button type="button" class="nb-btn btn-outline btn-sm" {% if next_enabled %}
hx-get="/knowledge?page={{ pagination.next_page }}{{ query_suffix }}" {% else %} disabled {% endif %}
hx-target="#knowledge_pane" hx-swap="outerHTML" hx-push-url="true">
Next
</button>
</div>
</div>
{% endfor %}
{% endif %}
</div>

28
html-router/templates/knowledge/relationship_table.html
View File

@@ -1,12 +1,11 @@
<div id="relationship_table_section"
class="overflow-x-auto shadow rounded-box border border-base-content/5 bg-base-100">
<table class="table">
<div id="relationship_table_section" class="overflow-x-auto nb-card mb-10">
<table class="nb-table">
<thead>
<tr>
<th>Origin</th>
<th>Target</th>
<th>Type</th>
<th>Actions</th>
<th class="text-left">Origin</th>
<th class="text-left">Target</th>
<th class="text-left">Type</th>
<th class="text-left">Actions</th>
</tr>
</thead>
<tbody>
@@ -31,9 +30,9 @@
{{ relationship.out }}
{% endfor %}
</td>
<td>{{ relationship.metadata.relationship_type }}</td>
<td class="uppercase tracking-wide text-xs">{{ relationship.metadata.relationship_type }}</td>
<td>
<button class="btn btn-sm btn-outline" hx-delete="/knowledge-relationship/{{ relationship.id }}"
<button class="nb-btn btn-xs" hx-delete="/knowledge-relationship/{{ relationship.id }}"
hx-target="#relationship_table_section" hx-swap="outerHTML">
{% include "icons/delete_icon.html" %}
</button>
@@ -43,7 +42,7 @@
<!-- New linking row -->
<tr id="new_relationship">
<td>
<select name="in_" class="select select-bordered w-full new_relationship_input">
<select name="in_" class="nb-select w-full new_relationship_input">
<option disabled selected>Select Origin</option>
{% for entity in entities %}
<option value="{{ entity.id }}">
@@ -53,7 +52,7 @@
</select>
</td>
<td>
<select name="out" class="select select-bordered w-full new_relationship_input">
<select name="out" class="nb-select w-full new_relationship_input">
<option disabled selected>Select Target</option>
{% for entity in entities %}
<option value="{{ entity.id }}">{{ entity.name }}</option>
@@ -62,12 +61,11 @@
</td>
<td>
<input id="relationship_type_input" name="relationship_type" type="text" placeholder="RelatedTo"
class="input input-bordered w-full new_relationship_input" />
class="nb-input w-full new_relationship_input" />
</td>
<td>
<button id="save_relationship_button" type="button" class="btn btn-primary btn-sm"
hx-post="/knowledge-relationship" hx-target="#relationship_table_section" hx-swap="outerHTML"
hx-include=".new_relationship_input">
<button id="save_relationship_button" type="button" class="nb-btn btn-sm" hx-post="/knowledge-relationship"
hx-target="#relationship_table_section" hx-swap="outerHTML" hx-include=".new_relationship_input">
Save
</button>
</td>

17
html-router/templates/modal_base.html
View File

@@ -1,21 +1,18 @@
<dialog id="body_modal" class="modal">
<div class="modal-box {% block modal_class %}{% endblock %} ">
<div class="modal-box rounded-none border-2 border-neutral bg-base-100 shadow-[8px_8px_0_0_#000] {% block modal_class %}{% endblock %}">
<form id="modal_form" {% block form_attributes %}{% endblock %}>
<div class="flex flex-col flex-1 space-y-4">
{% block modal_content %} <!-- Form fields go here in child templates -->
{% endblock %}
<div class="flex flex-col flex-1 gap-4">
{% block modal_content %}{% endblock %}
</div>
<div class="modal-action">
<div class="u-hairline mt-4 pt-3 flex flex-col gap-2 sm:flex-row sm:justify-end sm:items-center">
<!-- Close button (always visible) -->
<button type="button" class="btn" onclick="document.getElementById('body_modal').close()">
<button type="button" class="nb-btn w-full sm:w-auto" onclick="document.getElementById('body_modal').close()">
Close
</button>
<!-- Primary actions block -->
{% block primary_actions %}
<!-- Submit/Save buttons go here in child templates -->
{% endblock %}
{% block primary_actions %}{% endblock %}
</div>
</form>
</div>
@@ -38,4 +35,4 @@
<form method="dialog" class="modal-backdrop">
<button>close</button>
</form>
</dialog>
</dialog>

4
html-router/templates/navigation_bar.html
View File

@@ -1,4 +1,4 @@
<nav class="bg-base-200 sticky top-0 z-10">
<nav class="sticky top-0 z-10 nb-panel nb-panel-canvas border-t-0">
<div class="container mx-auto navbar">
<div class="mr-2 flex-1">
{% include "searchbar.html" %}
@@ -12,4 +12,4 @@
</ul>
</div>
</div>
</nav>
</nav>

8
html-router/templates/search/base.html
View File

@@ -5,7 +5,13 @@
{% block main %}
<div class="flex justify-center grow mt-2 sm:mt-4">
<div class="container">
<section class="mb-4">
<div class="nb-panel p-3 flex items-center justify-between">
<h1 class="text-xl font-extrabold tracking-tight">Search</h1>
<div class="text-xs opacity-70">Find documents, entities, and snippets</div>
</div>
</section>
{% include "search/response.html" %}
</div>
</div>
{% endblock %}
{% endblock %}

64
html-router/templates/search/response.html
View File

@@ -1,8 +1,8 @@
{% if search_result is defined and search_result %}
<ul class="list shadow">
<ul class="nb-card p-0">
{% for result in search_result %}
<li class="list-row hover:bg-base-200/50 p-4">
<div class="w-10 h-10 flex-shrink-0 mr-4 self-start mt-1">
<li class="p-4 u-hairline hover:bg-base-200/40 flex gap-3">
<div class="w-10 h-10 flex-shrink-0 self-start mt-1 grid place-items-center border-2 border-neutral bg-base-100 shadow-[4px_4px_0_0_#000]">
{% if result.url_info and result.url_info.url %}
<div class="tooltip tooltip-right" data-tip="Web Link">
{% include "icons/link_icon.html" %}
@@ -17,10 +17,10 @@
</div>
{% endif %}
</div>
<div class="flex-grow min-w-0">
<h3 class="text-lg font-semibold mb-1">
<a hx-get="/content/{{ result.id }}/read" hx-target="#modal" hx-swap="innerHTML"
class="link link-hover link-primary">
<div class="flex-1 min-w-0">
<h3 class="text-lg font-extrabold mb-1 leading-snug">
<a hx-get="/content/{{ result.id }}/read" hx-target="#modal" hx-swap="innerHTML" class="nb-link">
{% set title_text = result.highlighted_url_title
| default(result.url_info.title if result.url_info else none, true)
| default(result.highlighted_file_name, true)
@@ -30,8 +30,7 @@
</a>
</h3>
<div class="markdown-content prose prose-sm text-sm text-base-content/80 mb-3 overflow-hidden line-clamp-6"
data-content="{{result.highlighted_text | escape}}">
<div class="markdown-content prose-tufte-compact text-base-content/80 mb-3 overflow-hidden line-clamp-6" data-content="{{result.highlighted_text | escape}}">
{% if result.highlighted_text %}
{{ result.highlighted_text | escape }}
{% elif result.text %}
@@ -41,43 +40,46 @@
{% endif %}
</div>
<div class="text-xs text-base-content/70 flex flex-wrap gap-x-4 gap-y-1 items-center">
<span class="inline-flex items-center"><strong class="font-medium mr-1">Category:</strong>
<span class="badge badge-soft badge-secondary badge-sm">{{ result.highlighted_category |
default(result.category, true) |
safe }}</span>
<div class="text-xs flex flex-wrap gap-x-4 gap-y-2 items-center">
<span class="inline-flex items-center">
<span class="uppercase tracking-wide opacity-60 mr-2">Category</span>
<span class="nb-badge">{{ result.highlighted_category | default(result.category, true) | safe }}</span>
</span>
{% if result.highlighted_context or result.context %}
<span class="inline-flex items-center"><strong class="font-medium mr-1">Context:</strong>
<span class="badge badge-sm badge-outline">{{ result.highlighted_context | default(result.context, true) |
safe }}</span>
<span class="inline-flex items-center min-w-0">
<span class="uppercase tracking-wide opacity-60 mr-2">Context</span>
<span class="nb-badge">{{ result.highlighted_context | default(result.context, true) | safe }}</span>
</span>
{% endif %}
{% if result.url_info and result.url_info.url %}
<span class="inline-flex items-center min-w-0"><strong class="font-medium mr-1">Source:</strong>
<a href="{{ result.url_info.url }}" target="_blank" class="link link-hover link-xs truncate"
title="{{ result.url_info.url }}">
<span class="inline-flex items-center min-w-0">
<span class="uppercase tracking-wide opacity-60 mr-2">Source</span>
<a href="{{ result.url_info.url }}" target="_blank" class="nb-link truncate" title="{{ result.url_info.url }}">
{{ result.highlighted_url | default(result.url_info.url ) | safe }}
</a>
</span>
{% endif %}
<span class="badge badge-ghost badge-sm">Score: {{ result.score }}</span>
<span class="inline-flex items-center">
<span class="uppercase tracking-wide opacity-60 mr-2">Score</span>
<span class="nb-badge">{{ result.score }}</span>
</span>
</div>
</div>
</li>
{% endfor %}
</ul>
</ul>
{% elif query_param is defined and query_param | trim != "" %}
<div class="p-4 text-center text-base-content/70">
<p class="text-xl font-semibold mb-2">No results found for "<strong>{{ query_param | escape }}</strong>".</p>
<p class="text-sm">Try using different keywords or checking for typos.</p>
</div>
<div class="nb-panel p-5 text-center">
<p class="text-xl font-extrabold mb-2">No results for “{{ query_param | escape }}”.</p>
<p class="text-sm opacity-70">Try different keywords or check for typos.</p>
</div>
{% else %}
<div class="p-4 text-center text-base-content/70">
<p class="text-lg font-medium">Enter a term above to search your knowledge base.</p>
<p class="text-sm">Results will appear here.</p>
</div>
{% endif %}
<div class="nb-panel p-5 text-center">
<p class="text-lg font-semibold">Enter a term above to search your knowledge base.</p>
<p class="text-sm opacity-70">Results will appear here.</p>
</div>
{% endif %}

16
html-router/templates/searchbar.html
View File

@@ -1,8 +1,14 @@
<div class="flex items-center gap-2 min-w-[90px]">
<form class="w-full" hx-boost="true" method="get" action="/search"
<div class="flex items-center gap-2 min-w-[90px] w-full">
<form class="w-full relative" hx-boost="true" method="get" action="/search"
hx-trigger="keyup changed delay:500ms from:#search-input, search from:#search-input" hx-push-url="true">
<input id="search-input" type="search" placeholder="Search for anything..."
class="input input-sm input-bordered input-primary w-full" name="query" autocomplete="off"
value="{{ query_param | default('', true) }}" />
<input id="search-input" type="search" aria-label="Search" class=" nb-input w-full pl-9 ml-2" name="query"
autocomplete="off" value="{{ query_param | default('', true) }}" />
<button type="submit"
class="absolute right-1 top-1/2 -translate-y-1/2 nb-btn btn-xs px-3 h-7 bg-base-100 hover:bg-base-200">
Search
</button>
<span class="hidden md:inline absolute right-24 top-1/2 -translate-y-1/2 text-xs opacity-60">
press <kbd class="kbd kbd-xs">Enter</kbd>
</span>
</form>
</div>

48
html-router/templates/sidebar.html
View File

@@ -15,11 +15,12 @@
<div class="drawer-side z-20">
<label for="my-drawer" aria-label="close sidebar" class="drawer-overlay"></label>
<ul class="menu p-0 w-72 h-full bg-base-200 text-base-content flex flex-col">
<!-- <a class="px-2 mt-4 text-center text-2xl text-primary font-bold" href="/" hx-boost="true">Minne</a> -->
<ul class="menu p-0 w-72 h-full nb-canvas text-base-content flex flex-col border-r-2 border-neutral">
<!-- <a class="px-4 py-4 text-2xl font-extrabold tracking-tight text-primary border-b-2 border-neutral bg-base-100 nb-shadow" -->
<!-- href="/" hx-boost="true">Minne</a> -->
<!-- === TOP FIXED SECTION === -->
<div class="px-2 mt-14">
<div class="px-2 mt-4 space-y-3">
{% for url, name, label in [
("/", "home", "Dashboard"),
("/knowledge", "book", "Knowledge"),
@@ -28,22 +29,22 @@
("/search", "search", "Search")
] %}
<li>
<a hx-boost="true" href="{{ url }}" class="flex items-center gap-3">
<a hx-boost="true" href="{{ url }}" class="nb-btn w-full justify-start gap-3 bg-base-100 hover:bg-base-200">
{{ icon(name) }}
<span>{{ label }}</span>
<span class="uppercase tracking-wide">{{ label }}</span>
</a>
</li>
{% endfor %}
<li>
<button class="btn btn-primary btn-outline w-full flex items-center gap-3 justify-start mt-2"
hx-get="/ingress-form" hx-target="#modal" hx-swap="innerHTML">{% include "icons/send_icon.html" %} Add
<button class="nb-btn nb-cta w-full flex items-center gap-3 justify-start mt-2" hx-get="/ingress-form"
hx-target="#modal" hx-swap="innerHTML">{% include "icons/send_icon.html" %} Add
Content</button>
</li>
<div class="divider "></div>
<div class="u-hairline mt-4"></div>
</div>
<!-- === MIDDLE SCROLLABLE SECTION === -->
<span class="menu-title pb-4 ">Recent Chats</span>
<span class="px-4 py-2 font-semibold tracking-wide">Recent Chats</span>
<div class="flex-1 overflow-y-auto space-y-1 custom-scrollbar">
{% if conversation_archive is defined and conversation_archive %}
{% for conversation in conversation_archive %}
@@ -51,12 +52,12 @@
{% if edit_conversation_id == conversation.id %}
<!-- Edit mode -->
<form hx-patch="/chat/{{ conversation.id }}/title" hx-target=".drawer-side" hx-swap="outerHTML"
class="flex items-center gap-1 px-2 py-2">
<input type="text" name="title" value="{{ conversation.title }}" class="input input-sm flex-grow" />
<div class="flex gap-0.5">
<button type="submit" class="btn btn-ghost btn-xs">{% include "icons/check_icon.html" %}</button>
class="flex items-center gap-1 px-2 py-2 max-w-72 relative">
<input type="text" name="title" value="{{ conversation.title }}" class="nb-input nb-input-sm max-w-52" />
<div class="flex gap-0.5 absolute right-2">
<button type="submit" class="btn btn-ghost btn-xs !p-0">{% include "icons/check_icon.html" %}</button>
<button type="button" hx-get="/chat/sidebar" hx-target=".drawer-side" hx-swap="outerHTML"
class="btn btn-ghost btn-xs">
class="btn btn-ghost btn-xs !p-0">
{% include "icons/x_icon.html" %}
</button>
</div>
@@ -86,29 +87,30 @@
</div>
<!-- === BOTTOM FIXED SECTION === -->
<div class="px-2 pb-4">
<div class="divider "></div>
<div class="px-2 pb-4 space-y-3">
<li>
<a hx-boost="true" href="/account" class="flex btn btn-ghost justify-start items-center gap-3">
<a hx-boost="true" href="/account"
class="nb-btn w-full justify-start items-center gap-3 bg-base-100 hover:bg-base-200">
{% include "icons/user_icon.html" %}
<span>Account</span>
<span class="uppercase tracking-wide">Account</span>
</a>
</li>
{% if user.admin %}
<li>
<a hx-boost="true" href="/admin" class="flex btn btn-ghost justify-start items-center gap-3">
<a hx-boost="true" href="/admin"
class="nb-btn w-full justify-start items-center gap-3 bg-base-100 hover:bg-base-200">
{% include "icons/wrench_screwdriver_icon.html" %}
<span>Admin</span>
<span class="uppercase tracking-wide">Admin</span>
</a>
</li>
{% endif %}
<li>
<a hx-boost="true" href="/signout"
class="btn btn-error btn-outline w-full flex items-center gap-3 justify-start !mt-2">
class="nb-btn w-full justify-start items-center gap-3 bg-base-100 hover:bg-base-200 border-error text-error">
{% include "icons/logout_icon.html" %}
<span>Logout</span>
<span class="uppercase tracking-wide">Logout</span>
</a>
</li>
</div>
</ul>
</div>
</div>

19
ingestion-pipeline/src/pipeline.rs
View File

@@ -1,8 +1,9 @@
use std::{sync::Arc, time::Instant};
use chrono::Utc;
use futures::future::try_join_all;
use text_splitter::TextSplitter;
use tracing::{debug, info};
use tracing::info;
use common::{
error::AppError,
@@ -134,20 +135,22 @@ impl IngestionPipeline {
entities: Vec<KnowledgeEntity>,
relationships: Vec<KnowledgeRelationship>,
) -> Result<(), AppError> {
for entity in &entities {
debug!("Storing entity: {:?}", entity);
self.db.store_item(entity.clone()).await?;
}
let entity_count = entities.len();
let relationship_count = relationships.len();
let entity_futures = entities
.iter()
.map(|entitity| self.db.store_item(entitity.to_owned()));
try_join_all(entity_futures).await?;
for relationship in &relationships {
debug!("Storing relationship: {:?}", relationship);
relationship.store_relationship(&self.db).await?;
}
info!(
"Stored {} entities and {} relationships",
entities.len(),
relationships.len()
entity_count, relationship_count
);
Ok(())
}

12
ingestion-pipeline/src/types/mod.rs
View File

@@ -22,7 +22,9 @@ use std::io::{Seek, SeekFrom};
use tempfile::NamedTempFile;
use tracing::{error, info};
use crate::utils::image_parsing::extract_text_from_image;
use crate::utils::{
audio_transcription::transcribe_audio_file, image_parsing::extract_text_from_image,
};
pub async fn to_text_content(
ingestion_payload: IngestionPayload,
@@ -37,7 +39,7 @@ pub async fn to_text_content(
category,
user_id,
} => {
let (article, file_info) = fetch_article_from_url(&url, db, &user_id, &config).await?;
let (article, file_info) = fetch_article_from_url(&url, db, &user_id, config).await?;
Ok(TextContent::new(
article.text_content.into(),
Some(context),
@@ -179,7 +181,7 @@ async fn fetch_article_from_url(
};
// Store screenshot
let file_info = FileInfo::new(field_data, db, user_id, &config).await?;
let file_info = FileInfo::new(field_data, db, user_id, config).await?;
// Parse content...
let config = dom_smoothie::Config {
@@ -231,6 +233,10 @@ async fn extract_text_from_file(
let content = tokio::fs::read_to_string(&file_info.path).await?;
Ok(content)
}
"audio/mpeg" | "audio/mp3" | "audio/wav" | "audio/x-wav" | "audio/webm" | "audio/mp4"
| "audio/ogg" | "audio/flac" => {
transcribe_audio_file(&file_info.path, db_client, openai_client).await
}
// Handle other MIME types as needed
_ => Err(AppError::NotFound(file_info.mime_type.clone())),
}

28
ingestion-pipeline/src/utils/audio_transcription.rs Normal file
View File

@@ -0,0 +1,28 @@
use async_openai::types::{AudioResponseFormat, CreateTranscriptionRequestArgs};
use common::{
error::AppError,
storage::{db::SurrealDbClient, types::system_settings::SystemSettings},
};
/// Transcribes an audio file using the configured OpenAI Whisper model.
pub async fn transcribe_audio_file(
file_path: &str,
db_client: &SurrealDbClient,
openai_client: &async_openai::Client<async_openai::config::OpenAIConfig>,
) -> Result<String, AppError> {
let system_settings = SystemSettings::get_current(db_client).await?;
let model = system_settings.voice_processing_model;
let request = CreateTranscriptionRequestArgs::default()
.file(file_path)
.model(model)
.response_format(AudioResponseFormat::Json)
.build()?;
let response = openai_client
.audio()
.transcribe(request)
.await
.map_err(|e| AppError::Processing(format!("Audio transcription failed: {}", e)))?;
Ok(response.text)
}

2
ingestion-pipeline/src/utils/image_parsing.rs
View File

@@ -48,7 +48,7 @@ pub async fn extract_text_from_image(
let description = response
.choices
.get(0)
.first()
.and_then(|c| c.message.content.as_ref())
.cloned()
.unwrap_or_else(|| "No description found.".to_string());

19
ingestion-pipeline/src/utils/mod.rs
View File

@@ -1,3 +1,4 @@
pub mod audio_transcription;
pub mod image_parsing;
pub mod llm_instructions;
@@ -32,16 +33,12 @@ impl GraphMapper {
}
// If parsing fails, look it up in the map.
self.key_to_id
.get(key)
.map(|id| *id) // Dereference the &Uuid to get Uuid
// If `get` returned None, create and return an error.
.ok_or_else(|| {
AppError::GraphMapper(format!(
"Key '{}' is not a valid UUID and was not found in the map.",
key
))
})
self.key_to_id.get(key).copied().ok_or_else(|| {
AppError::GraphMapper(format!(
"Key '{}' is not a valid UUID and was not found in the map.",
key
))
})
}
/// Assigns a new UUID for a given key. (No changes needed here)
@@ -55,7 +52,7 @@ impl GraphMapper {
pub fn get_id(&self, key: &str) -> Result<Uuid, AppError> {
self.key_to_id
.get(key)
.map(|id| *id)
.copied()
.ok_or_else(|| AppError::GraphMapper(format!("Key '{}' not found in map.", key)))
}
}

11
json-stream-parser/src/lib.rs
View File

@@ -441,11 +441,9 @@ pub fn parse_stream(json_string: &str) -> Result<Value, String> {
current_status.clone(),
current_char.to_string()
);
if let Err(e) = add_char_into_object(&mut out, &mut current_status, current_char) {
return Err(e);
}
add_char_into_object(&mut out, &mut current_status, current_char)?
}
return Ok(out);
Ok(out)
}
#[cfg(not(debug_assertions))]
@@ -464,6 +462,11 @@ pub struct JsonStreamParser {
object: Value,
current_status: ObjectStatus,
}
impl Default for JsonStreamParser {
fn default() -> Self {
Self::new()
}
}
impl JsonStreamParser {
pub fn new() -> JsonStreamParser {

3
main/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "main"
version = "0.1.4"
version = "0.2.1"
edition = "2021"
repository = "https://github.com/perstarkse/minne"
license = "AGPL-3.0-or-later"
@@ -34,3 +34,4 @@ path = "src/worker.rs"
[[bin]]
name = "main"
path = "src/main.rs"

2
main/src/main.rs
View File

@@ -13,7 +13,7 @@ use tokio::task::LocalSet;
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Set up tracing
tracing_subscriber::registry()
.with(fmt::layer())
.with(fmt::layer().with_writer(std::io::stderr))
.with(EnvFilter::from_default_env())
.try_init()
.ok();

Some files were not shown because too many files have changed in this diff Show More