starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-03-26 03:11:34 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

release: 1.0.0

Browse Source
This commit is contained in:
Per Stark
2026-01-11 18:37:07 +01:00
parent db43be1606
commit 8fe4ac9fec
53 changed files with 757 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
common/migrations/20251231_enforce_schemafull.surql
View File

@@ -62,7 +62,13 @@ DEFINE TABLE OVERWRITE conversation SCHEMAFULL;
DEFINE TABLE OVERWRITE file SCHEMAFULL;
DEFINE TABLE OVERWRITE knowledge_entity SCHEMAFULL;
DEFINE TABLE OVERWRITE message SCHEMAFULL;
DEFINE TABLE OVERWRITE relates_to SCHEMAFULL;
DEFINE TABLE OVERWRITE relates_to SCHEMAFULL TYPE RELATION;
DEFINE FIELD IF NOT EXISTS in ON relates_to TYPE record<knowledge_entity>;
DEFINE FIELD IF NOT EXISTS out ON relates_to TYPE record<knowledge_entity>;
DEFINE FIELD IF NOT EXISTS metadata ON relates_to TYPE object;
DEFINE FIELD IF NOT EXISTS metadata.user_id ON relates_to TYPE string;
DEFINE FIELD IF NOT EXISTS metadata.source_id ON relates_to TYPE string;
DEFINE FIELD IF NOT EXISTS metadata.relationship_type ON relates_to TYPE string;
DEFINE TABLE OVERWRITE scratchpad SCHEMAFULL;
DEFINE TABLE OVERWRITE system_settings SCHEMAFULL;
DEFINE TABLE OVERWRITE text_chunk SCHEMAFULL;

6
common/src/storage/indexes.rs
View File

@@ -253,11 +253,7 @@ async fn ensure_runtime_indexes_inner(
Ok(())
}
async fn get_index_status(
db: &SurrealDbClient,
index_name: &str,
table: &str,
) -> Result<String> {
async fn get_index_status(db: &SurrealDbClient, index_name: &str, table: &str) -> Result<String> {
let info_query = format!("INFO FOR INDEX {index_name} ON TABLE {table};");
let mut info_res = db
.client

5
common/src/storage/types/file_info.rs
View File

@@ -3,7 +3,10 @@ use bytes;
use mime_guess::from_path;
use object_store::Error as ObjectStoreError;
use sha2::{Digest, Sha256};
use std::{io::{BufReader, Read}, path::Path};
use std::{
io::{BufReader, Read},
path::Path,
};
use tempfile::NamedTempFile;
use thiserror::Error;
use tokio::task;

15
common/src/storage/types/knowledge_entity.rs
View File

@@ -460,7 +460,11 @@ impl KnowledgeEntity {
for (i, entity) in all_entities.iter().enumerate() {
if i > 0 && i % 100 == 0 {
info!(progress = i, total = total_entities, "Re-embedding progress");
info!(
progress = i,
total = total_entities,
"Re-embedding progress"
);
}
let embedding_input = format!(
@@ -489,12 +493,12 @@ impl KnowledgeEntity {
// Clear existing embeddings and index first to prevent SurrealDB panics and dimension conflicts.
info!("Removing old index and clearing embeddings...");
// Explicitly remove the index first. This prevents background HNSW maintenance from crashing
// when we delete/replace data, dealing with a known SurrealDB panic.
db.client
.query(format!(
"REMOVE INDEX idx_embedding_knowledge_entity_embedding ON TABLE {};",
"REMOVE INDEX idx_embedding_knowledge_entity_embedding ON TABLE {};",
KnowledgeEntityEmbedding::table_name()
))
.await
@@ -503,7 +507,10 @@ impl KnowledgeEntity {
.map_err(AppError::Database)?;
db.client
.query(format!("DELETE FROM {};", KnowledgeEntityEmbedding::table_name()))
.query(format!(
"DELETE FROM {};",
KnowledgeEntityEmbedding::table_name()
))
.await
.map_err(AppError::Database)?
.check()

24
common/src/storage/types/knowledge_relationship.rs
View File

@@ -55,7 +55,7 @@ impl KnowledgeRelationship {
relationship_type = self.metadata.relationship_type.as_str()
);
db_client.query(query).await?;
db_client.query(query).await?.check()?;
Ok(())
}
@@ -99,9 +99,7 @@ impl KnowledgeRelationship {
Err(AppError::NotFound(format!("Relationship {id} not found")))
}
} else {
db_client
.query(format!("DELETE relates_to:`{id}`"))
.await?;
db_client.query(format!("DELETE relates_to:`{id}`")).await?;
Ok(())
}
}
@@ -161,7 +159,7 @@ mod tests {
}
#[tokio::test]
async fn test_store_relationship() {
async fn test_store_and_verify_by_source_id() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();
@@ -169,6 +167,10 @@ mod tests {
.await
.expect("Failed to start in-memory surrealdb");
db.apply_migrations()
.await
.expect("Failed to apply migrations");
// Create two entities to relate
let entity1_id = create_test_entity("Entity 1", &db).await;
let entity2_id = create_test_entity("Entity 2", &db).await;
@@ -209,7 +211,7 @@ mod tests {
}
#[tokio::test]
async fn test_delete_relationship_by_id() {
async fn test_store_and_delete_relationship() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();
@@ -234,7 +236,7 @@ mod tests {
relationship_type,
);
// Store the relationship
// Store relationship
relationship
.store_relationship(&db)
.await
@@ -255,12 +257,12 @@ mod tests {
"Relationship should exist before deletion"
);
// Delete the relationship by ID
// Delete relationship by ID
KnowledgeRelationship::delete_relationship_by_id(&relationship.id, &user_id, &db)
.await
.expect("Failed to delete relationship by ID");
// Query to verify the relationship was deleted
// Query to verify relationship was deleted
let mut result = db
.query(format!(
"SELECT * FROM relates_to WHERE metadata.user_id = '{}' AND metadata.source_id = '{}'",
@@ -270,7 +272,7 @@ mod tests {
.expect("Query failed");
let results: Vec<KnowledgeRelationship> = result.take(0).unwrap_or_default();
// Verify the relationship no longer exists
// Verify relationship no longer exists
assert!(results.is_empty(), "Relationship should be deleted");
}
@@ -342,7 +344,7 @@ mod tests {
}
#[tokio::test]
async fn test_delete_relationships_by_source_id() {
async fn test_store_relationship_exists() {
// Setup in-memory database for testing
let namespace = "test_ns";
let database = &Uuid::new_v4().to_string();

2
common/src/storage/types/mod.rs
View File

@@ -116,7 +116,7 @@ macro_rules! stored_object {
}
$(#[$struct_attr])*
$(#[$struct_attr])*
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct $name {
#[serde(deserialize_with = "deserialize_flexible_id")]

8
common/src/storage/types/text_chunk.rs
View File

@@ -352,12 +352,12 @@ impl TextChunk {
// Generate all new embeddings in memory
let mut new_embeddings: HashMap<String, (Vec<f32>, String, String)> = HashMap::new();
info!("Generating new embeddings for all chunks...");
for (i, chunk) in all_chunks.iter().enumerate() {
if i > 0 && i % 100 == 0 {
info!(progress = i, total = total_chunks, "Re-embedding progress");
}
let embedding = provider
.embed(&chunk.chunk)
.await
@@ -381,12 +381,12 @@ impl TextChunk {
// Clear existing embeddings and index first to prevent SurrealDB panics and dimension conflicts.
info!("Removing old index and clearing embeddings...");
// Explicitly remove the index first. This prevents background HNSW maintenance from crashing
// when we delete/replace data, dealing with a known SurrealDB panic.
db.client
.query(format!(
"REMOVE INDEX idx_embedding_text_chunk_embedding ON TABLE {};",
"REMOVE INDEX idx_embedding_text_chunk_embedding ON TABLE {};",
TextChunkEmbedding::table_name()
))
.await

5
common/src/utils/embedding.rs
View File

@@ -250,9 +250,8 @@ impl EmbeddingProvider {
match config.embedding_backend {
EmbeddingBackend::OpenAI => {
let client = openai_client.ok_or_else(|| {
anyhow!("OpenAI embedding backend requires an OpenAI client")
})?;
let client = openai_client
.ok_or_else(|| anyhow!("OpenAI embedding backend requires an OpenAI client"))?;
// Use defaults that match SystemSettings initial values
Self::new_openai(client, "text-embedding-3-small".to_string(), 1536)
}