working storing relationships

This commit is contained in:
Per Stark
2024-11-06 21:24:30 +01:00
parent 6d8cd05c1a
commit 3f06bf969a
3 changed files with 120 additions and 115 deletions
+1
View File
@@ -51,6 +51,7 @@ impl From<String> for KnowledgeEntityType {
/// Represents a relationship between two knowledge entities. /// Represents a relationship between two knowledge entities.
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
pub struct KnowledgeRelationship { pub struct KnowledgeRelationship {
#[serde(deserialize_with = "thing_to_string")]
pub id: String, pub id: String,
#[serde(rename = "in")] #[serde(rename = "in")]
pub in_: String, // Target KnowledgeEntity ID pub in_: String, // Target KnowledgeEntity ID
+6 -6
View File
@@ -84,13 +84,13 @@ impl TextContent {
for relationship in relationships { for relationship in relationships {
// info!("{:?}", relationship); // info!("{:?}", relationship);
// let _created: Option<Record> = db_client let _created: Option<KnowledgeRelationship> = db_client
// .client .client
// .insert(("knowledge_relationship", &relationship.id.to_string())) .insert(("knowledge_relationship", &relationship.id.to_string()))
// .content(relationship) .content(relationship)
// .await?; .await?;
// debug!("{:?}",_created); debug!("{:?}",_created);
} }
+25 -21
View File
@@ -1,20 +1,14 @@
use async_openai::types::ChatCompletionRequestSystemMessage;
use async_openai::types::ChatCompletionRequestUserMessage;
use async_openai::types::CreateChatCompletionRequestArgs;
use serde::Deserialize;
use serde::Serialize;
use surrealdb::sql::Thing;
use surrealdb::RecordId;
use tracing::debug;
use tracing::info;
use uuid::Uuid;
use crate::models::graph_entities::GraphMapper; use crate::models::graph_entities::GraphMapper;
use crate::models::graph_entities::KnowledgeEntity; use crate::models::graph_entities::KnowledgeEntity;
use crate::models::graph_entities::KnowledgeEntityType; use crate::models::graph_entities::KnowledgeEntityType;
use crate::models::graph_entities::KnowledgeRelationship; use crate::models::graph_entities::KnowledgeRelationship;
use crate::models::text_content::ProcessingError; use crate::models::text_content::ProcessingError;
use crate::surrealdb::SurrealDbClient; use crate::surrealdb::SurrealDbClient;
use async_openai::types::{CreateChatCompletionRequestArgs, ChatCompletionRequestUserMessage, ChatCompletionRequestSystemMessage };
use serde::{Deserialize, Serialize};
use serde_json::json; use serde_json::json;
use tracing::{info,debug};
use uuid::Uuid;
/// Represents a single knowledge entity from the LLM. /// Represents a single knowledge entity from the LLM.
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -42,11 +36,15 @@ pub struct LLMGraphAnalysisResult {
} }
impl LLMGraphAnalysisResult { impl LLMGraphAnalysisResult {
pub fn to_database_entities(&self, source_id: &Uuid) -> (Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>) { pub fn to_database_entities(
&self,
source_id: &Uuid,
) -> (Vec<KnowledgeEntity>, Vec<KnowledgeRelationship>) {
let mut mapper = GraphMapper::new(); let mut mapper = GraphMapper::new();
// First pass: Create all entities and map their keys to UUIDs // First pass: Create all entities and map their keys to UUIDs
let entities: Vec<KnowledgeEntity> = self.knowledge_entities let entities: Vec<KnowledgeEntity> = self
.knowledge_entities
.iter() .iter()
.map(|llm_entity| { .map(|llm_entity| {
let id = mapper.assign_id(&llm_entity.key); let id = mapper.assign_id(&llm_entity.key);
@@ -62,7 +60,8 @@ impl LLMGraphAnalysisResult {
.collect(); .collect();
// Second pass: Create relationships using mapped UUIDs // Second pass: Create relationships using mapped UUIDs
let relationships: Vec<KnowledgeRelationship> = self.relationships let relationships: Vec<KnowledgeRelationship> = self
.relationships
.iter() .iter()
.filter_map(|llm_rel| { .filter_map(|llm_rel| {
let source_id = mapper.get_id(&llm_rel.source)?; let source_id = mapper.get_id(&llm_rel.source)?;
@@ -83,7 +82,12 @@ impl LLMGraphAnalysisResult {
} }
/// Sends text to an LLM for analysis. /// Sends text to an LLM for analysis.
pub async fn create_json_ld(category: &str, instructions: &str, text: &str, db_client: &SurrealDbClient) -> Result<LLMGraphAnalysisResult, ProcessingError> { pub async fn create_json_ld(
category: &str,
instructions: &str,
text: &str,
db_client: &SurrealDbClient,
) -> Result<LLMGraphAnalysisResult, ProcessingError> {
// Get the nodes from the database // Get the nodes from the database
let entities: Vec<KnowledgeEntity> = db_client.client.select("knowledge_entity").await?; let entities: Vec<KnowledgeEntity> = db_client.client.select("knowledge_entity").await?;
info!("{:?}", entities); info!("{:?}", entities);
@@ -91,8 +95,6 @@ pub async fn create_json_ld(category: &str, instructions: &str, text: &str, db_c
let deleted: Vec<KnowledgeEntity> = db_client.client.delete("knowledge_entity").await?; let deleted: Vec<KnowledgeEntity> = db_client.client.delete("knowledge_entity").await?;
info! {"{:?}", deleted}; info! {"{:?}", deleted};
let client = async_openai::Client::new(); let client = async_openai::Client::new();
let schema = json!({ let schema = json!({
"type": "object", "type": "object",
@@ -179,7 +181,6 @@ pub async fn create_json_ld(category: &str, instructions: &str, text: &str, db_c
6. Only create relationships between existing KnowledgeEntities. 6. Only create relationships between existing KnowledgeEntities.
"#; "#;
let user_message = format!( let user_message = format!(
"Category: {}\nInstructions: {}\nContent:\n{}", "Category: {}\nInstructions: {}\nContent:\n{}",
category, instructions, text category, instructions, text
@@ -194,12 +195,15 @@ pub async fn create_json_ld(category: &str, instructions: &str, text: &str, db_c
ChatCompletionRequestUserMessage::from(user_message).into(), ChatCompletionRequestUserMessage::from(user_message).into(),
]) ])
.response_format(response_format) .response_format(response_format)
.build().map_err(|e| ProcessingError::LLMError(e.to_string()))?; .build()
.map_err(|e| ProcessingError::LLMError(e.to_string()))?;
// Send the request to OpenAI // Send the request to OpenAI
let response = client.chat().create(request).await.map_err(|e| { let response = client
ProcessingError::LLMError(format!("OpenAI API request failed: {}", e)) .chat()
})?; .create(request)
.await
.map_err(|e| ProcessingError::LLMError(format!("OpenAI API request failed: {}", e)))?;
debug!("{:?}", response); debug!("{:?}", response);