refactor: renamed instructions to context

common/migrations/20250503_215025_initial_setup.surql

@@ -14,6 +14,6 @@ CREATE system_settings:current CONTENT {
     query_model: "gpt-4o-mini",
     processing_model: "gpt-4o-mini",
     query_system_prompt: "You are a knowledgeable assistant with access to a specialized knowledge base. You will be provided with relevant knowledge entities from the database as context. Each knowledge entity contains a name, description, and type, representing different concepts, ideas, and information.\nYour task is to:\n1. Carefully analyze the provided knowledge entities in the context\n2. Answer user questions based on this information\n3. Provide clear, concise, and accurate responses\n4. When referencing information, briefly mention which knowledge entity it came from\n5. If the provided context doesn't contain enough information to answer the question confidently, clearly state this\n6. If only partial information is available, explain what you can answer and what information is missing\n7. Avoid making assumptions or providing information not supported by the context\n8. Output the references to the documents. Use the UUIDs and make sure they are correct!\nRemember:\n- Be direct and honest about the limitations of your knowledge\n- Cite the relevant knowledge entities when providing information, but only provide the UUIDs in the reference array\n- If you need to combine information from multiple entities, explain how they connect\n- Don't speculate beyond what's provided in the context\nExample response formats:\n\"Based on [Entity Name], [answer...]\"\n\"I found relevant information in multiple entries: [explanation...]\"\n\"I apologize, but the provided context doesn't contain information about [topic]\"",
-    ingestion_system_prompt: "You are an AI assistant. You will receive a text content, along with user instructions and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users instructions and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.\nThe JSON should have the following structure:\n{\n\"knowledge_entities\": [\n{\n\"key\": \"unique-key-1\",\n\"name\": \"Entity Name\",\n\"description\": \"A detailed description of the entity.\",\n\"entity_type\": \"TypeOfEntity\"\n},\n// More entities...\n],\n\"relationships\": [\n{\n\"type\": \"RelationshipType\",\n\"source\": \"unique-key-1 or UUID from existing database\",\n\"target\": \"unique-key-1 or UUID from existing database\"\n},\n// More relationships...\n]\n}\nGuidelines:\n1. Do NOT generate any IDs or UUIDs. Use a unique `key` for each knowledge entity.\n2. Each KnowledgeEntity should have a unique `key`, a meaningful `name`, and a descriptive `description`.\n3. Define the type of each KnowledgeEntity using the following categories: Idea, Project, Document, Page, TextSnippet.\n4. Establish relationships between entities using types like RelatedTo, RelevantTo, SimilarTo.\n5. Use the `source` key to indicate the originating entity and the `target` key to indicate the related entity\"\n6. You will be presented with a few existing KnowledgeEntities that are similar to the current ones. They will have an existing UUID. When creating relationships to these entities, use their UUID.\n7. Only create relationships between existing KnowledgeEntities.\n8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.\n9. A new relationship MUST include a newly created KnowledgeEntity."
+    ingestion_system_prompt: "You are an AI assistant. You will receive a text content, along with user context and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users context and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.\nThe JSON should have the following structure:\n{\n\"knowledge_entities\": [\n{\n\"key\": \"unique-key-1\",\n\"name\": \"Entity Name\",\n\"description\": \"A detailed description of the entity.\",\n\"entity_type\": \"TypeOfEntity\"\n},\n// More entities...\n],\n\"relationships\": [\n{\n\"type\": \"RelationshipType\",\n\"source\": \"unique-key-1 or UUID from existing database\",\n\"target\": \"unique-key-1 or UUID from existing database\"\n},\n// More relationships...\n]\n}\nGuidelines:\n1. Do NOT generate any IDs or UUIDs. Use a unique `key` for each knowledge entity.\n2. Each KnowledgeEntity should have a unique `key`, a meaningful `name`, and a descriptive `description`.\n3. Define the type of each KnowledgeEntity using the following categories: Idea, Project, Document, Page, TextSnippet.\n4. Establish relationships between entities using types like RelatedTo, RelevantTo, SimilarTo.\n5. Use the `source` key to indicate the originating entity and the `target` key to indicate the related entity\"\n6. You will be presented with a few existing KnowledgeEntities that are similar to the current ones. They will have an existing UUID. When creating relationships to these entities, use their UUID.\n7. Only create relationships between existing KnowledgeEntities.\n8. Entities that exist already in the database should NOT be created again. If there is only a minor overlap, skip creating a new entity.\n9. A new relationship MUST include a newly created KnowledgeEntity."
 };
 END;

common/migrations/20250509_152033_rename_instructions_to_context.surql

@@ -0,0 +1,8 @@
+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;
+DEFINE FIELD OVERWRITE instructions ON text_content TYPE option<string>;
+
+UPDATE text_content SET context = instructions WHERE instructions IS NOT NONE;
+
+UPDATE text_content UNSET instructions;
+
+REMOVE FIELD instructions ON TABLE text_content;

common/migrations/definitions/20250509_152033_rename_instructions_to_context.json

@@ -0,0 +1 @@
+{"schemas":"--- original\n+++ modified\n@@ -198,11 +198,11 @@\n DEFINE FIELD IF NOT EXISTS file_info ON text_content TYPE option<object>;\n # UrlInfo is a struct, store as object\n DEFINE FIELD IF NOT EXISTS url_info ON text_content TYPE option<object>;\n-DEFINE FIELD IF NOT EXISTS instructions ON text_content TYPE string;\n+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;\n DEFINE FIELD IF NOT EXISTS category ON text_content TYPE string;\n DEFINE FIELD IF NOT EXISTS user_id ON text_content TYPE string;\n\n-# Indexes based on query patterns (get_latest_text_contents, get_text_contents_by_category)\n+# Indexes based on query patterns\n DEFINE INDEX IF NOT EXISTS text_content_user_id_idx ON text_content FIELDS user_id;\n DEFINE INDEX IF NOT EXISTS text_content_created_at_idx ON text_content FIELDS created_at;\n DEFINE INDEX IF NOT EXISTS text_content_category_idx ON text_content FIELDS category;\n","events":null}

common/schemas/text_content.surql

@@ -12,11 +12,11 @@ DEFINE FIELD IF NOT EXISTS text ON text_content TYPE string;
 DEFINE FIELD IF NOT EXISTS file_info ON text_content TYPE option<object>;
 # UrlInfo is a struct, store as object
 DEFINE FIELD IF NOT EXISTS url_info ON text_content TYPE option<object>;
-DEFINE FIELD IF NOT EXISTS instructions ON text_content TYPE string;
+DEFINE FIELD IF NOT EXISTS context ON text_content TYPE option<string>;
 DEFINE FIELD IF NOT EXISTS category ON text_content TYPE string;
 DEFINE FIELD IF NOT EXISTS user_id ON text_content TYPE string;
 
-# Indexes based on query patterns (get_latest_text_contents, get_text_contents_by_category)
+# Indexes based on query patterns
 DEFINE INDEX IF NOT EXISTS text_content_user_id_idx ON text_content FIELDS user_id;
 DEFINE INDEX IF NOT EXISTS text_content_created_at_idx ON text_content FIELDS created_at;
 DEFINE INDEX IF NOT EXISTS text_content_category_idx ON text_content FIELDS category;

common/src/storage/types/ingestion_payload.rs

@@ -7,30 +7,30 @@ use url::Url;
 pub enum IngestionPayload {
     Url {
         url: String,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
     Text {
         text: String,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
     File {
         file_info: FileInfo,
-        instructions: String,
+        context: String,
         category: String,
         user_id: String,
     },
 }
 
 impl IngestionPayload {
-    /// Creates ingestion payloads from the provided content, instructions, and files.
+    /// Creates ingestion payloads from the provided content, context, and files.
     ///
     /// # Arguments
     /// * `content` - Optional textual content to be ingressed
-    /// * `instructions` - Instructions for processing the ingress content
+    /// * `context` - context for processing the ingress content
     /// * `category` - Category to classify the ingressed content
     /// * `files` - Vector of `FileInfo` objects containing information about uploaded files
     /// * `user_id` - Identifier of the user performing the ingress operation
@@ -40,7 +40,7 @@ impl IngestionPayload {
     ///   (one per file/content type). On failure, returns an `AppError`.
     pub fn create_ingestion_payload(
         content: Option<String>,
-        instructions: String,
+        context: String,
         category: String,
         files: Vec<FileInfo>,
         user_id: &str,
@@ -55,7 +55,7 @@ impl IngestionPayload {
                     info!("Detected URL: {}", url);
                     object_list.push(IngestionPayload::Url {
                         url: url.to_string(),
-                        instructions: instructions.clone(),
+                        context: context.clone(),
                         category: category.clone(),
                         user_id: user_id.into(),
                     });
@@ -65,7 +65,7 @@ impl IngestionPayload {
                         info!("Treating input as plain text");
                         object_list.push(IngestionPayload::Text {
                             text: input_content.to_string(),
-                            instructions: instructions.clone(),
+                            context: context.clone(),
                             category: category.clone(),
                             user_id: user_id.into(),
                         });
@@ -77,7 +77,7 @@ impl IngestionPayload {
         for file in files {
             object_list.push(IngestionPayload::File {
                 file_info: file,
-                instructions: instructions.clone(),
+                context: context.clone(),
                 category: category.clone(),
                 user_id: user_id.into(),
             })
@@ -126,14 +126,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_url() {
         let url = "https://example.com";
-        let instructions = "Process this URL";
+        let context = "Process this URL";
         let category = "websites";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(url.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -144,13 +144,13 @@ mod tests {
         match &result[0] {
             IngestionPayload::Url {
                 url: payload_url,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 // URL parser may normalize the URL by adding a trailing slash
                 assert!(payload_url == &url.to_string() || payload_url == &format!("{}/", url));
-                assert_eq!(payload_instructions, &instructions);
+                assert_eq!(payload_context, &context);
                 assert_eq!(payload_category, &category);
                 assert_eq!(payload_user_id, &user_id);
             }
@@ -161,14 +161,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_text() {
         let text = "This is some text content";
-        let instructions = "Process this text";
+        let context = "Process this text";
         let category = "notes";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(text.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -179,12 +179,12 @@ mod tests {
         match &result[0] {
             IngestionPayload::Text {
                 text: payload_text,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 assert_eq!(payload_text, text);
-                assert_eq!(payload_instructions, instructions);
+                assert_eq!(payload_context, context);
                 assert_eq!(payload_category, category);
                 assert_eq!(payload_user_id, user_id);
             }
@@ -194,7 +194,7 @@ mod tests {
 
     #[test]
     fn test_create_ingestion_payload_with_file() {
-        let instructions = "Process this file";
+        let context = "Process this file";
         let category = "documents";
         let user_id = "user123";
 
@@ -208,7 +208,7 @@ mod tests {
 
         let result = IngestionPayload::create_ingestion_payload(
             None,
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -219,12 +219,12 @@ mod tests {
         match &result[0] {
             IngestionPayload::File {
                 file_info: payload_file_info,
-                instructions: payload_instructions,
+                context: payload_context,
                 category: payload_category,
                 user_id: payload_user_id,
             } => {
                 assert_eq!(payload_file_info.id, file_info.id);
-                assert_eq!(payload_instructions, instructions);
+                assert_eq!(payload_context, context);
                 assert_eq!(payload_category, category);
                 assert_eq!(payload_user_id, user_id);
             }
@@ -235,7 +235,7 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_url_and_file() {
         let url = "https://example.com";
-        let instructions = "Process this data";
+        let context = "Process this data";
         let category = "mixed";
         let user_id = "user123";
 
@@ -249,7 +249,7 @@ mod tests {
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(url.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -283,14 +283,14 @@ mod tests {
 
     #[test]
     fn test_create_ingestion_payload_empty_input() {
-        let instructions = "Process something";
+        let context = "Process something";
         let category = "empty";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             None,
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,
@@ -308,14 +308,14 @@ mod tests {
     #[test]
     fn test_create_ingestion_payload_with_empty_text() {
         let text = ""; // Empty text
-        let instructions = "Process this";
+        let context = "Process this";
         let category = "notes";
         let user_id = "user123";
         let files = vec![];
 
         let result = IngestionPayload::create_ingestion_payload(
             Some(text.to_string()),
-            instructions.to_string(),
+            context.to_string(),
             category.to_string(),
             files,
             user_id,

common/src/storage/types/ingestion_task.rs

@@ -110,7 +110,7 @@ mod tests {
     fn create_test_payload(user_id: &str) -> IngestionPayload {
         IngestionPayload::Text {
             text: "Test content".to_string(),
-            instructions: "Test instructions".to_string(),
+            context: "Test context".to_string(),
             category: "Test category".to_string(),
             user_id: user_id.to_string(),
         }

common/src/storage/types/system_prompts.rs

@@ -20,7 +20,7 @@ Example response formats:
 "I found relevant information in multiple entries: [explanation...]"
 "I apologize, but the provided context doesn't contain information about [topic]""#;
 
-pub static DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT: &str = r#"You are an AI assistant. You will receive a text content, along with user instructions and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users instructions and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.
+pub static DEFAULT_INGRESS_ANALYSIS_SYSTEM_PROMPT: &str = r#"You are an AI assistant. You will receive a text content, along with user context and a category. Your task is to provide a structured JSON object representing the content in a graph format suitable for a graph database. You will also be presented with some existing knowledge_entities from the database, do not replicate these! Your task is to create meaningful knowledge entities from the submitted content. Try and infer as much as possible from the users context and category when creating these. If the user submits a large content, create more general entities. If the user submits a narrow and precise content, try and create precise knowledge entities.
 
 The JSON should have the following structure:
 

common/src/storage/types/text_content.rs

@@ -16,7 +16,7 @@ stored_object!(TextContent, "text_content", {
     text: String,
     file_info: Option<FileInfo>,
     url_info: Option<UrlInfo>,
-    instructions: String,
+    context: Option<String>,
     category: String,
     user_id: String
 });
@@ -24,7 +24,7 @@ stored_object!(TextContent, "text_content", {
 impl TextContent {
     pub fn new(
         text: String,
-        instructions: String,
+        context: Option<String>,
         category: String,
         file_info: Option<FileInfo>,
         url_info: Option<UrlInfo>,
@@ -38,7 +38,7 @@ impl TextContent {
             text,
             file_info,
             url_info,
-            instructions,
+            context,
             category,
             user_id,
         }
@@ -46,7 +46,7 @@ impl TextContent {
 
     pub async fn patch(
         id: &str,
-        instructions: &str,
+        context: &str,
         category: &str,
         text: &str,
         db: &SurrealDbClient,
@@ -55,7 +55,7 @@ impl TextContent {
 
         let _res: Option<Self> = db
             .update((Self::table_name(), id))
-            .patch(PatchOp::replace("/instructions", instructions))
+            .patch(PatchOp::replace("/context", context))
             .patch(PatchOp::replace("/category", category))
             .patch(PatchOp::replace("/text", text))
             .patch(PatchOp::replace("/updated_at", now))
@@ -73,13 +73,13 @@ mod tests {
     async fn test_text_content_creation() {
         // Test basic object creation
         let text = "Test content text".to_string();
-        let instructions = "Test instructions".to_string();
+        let context = "Test context".to_string();
         let category = "Test category".to_string();
         let user_id = "user123".to_string();
 
         let text_content = TextContent::new(
             text.clone(),
-            instructions.clone(),
+            Some(context.clone()),
             category.clone(),
             None,
             None,
@@ -88,7 +88,7 @@ mod tests {
 
         // Check that the fields are set correctly
         assert_eq!(text_content.text, text);
-        assert_eq!(text_content.instructions, instructions);
+        assert_eq!(text_content.context, Some(context));
         assert_eq!(text_content.category, category);
         assert_eq!(text_content.user_id, user_id);
         assert!(text_content.file_info.is_none());
@@ -100,7 +100,7 @@ mod tests {
     async fn test_text_content_with_url() {
         // Test creating with URL
         let text = "Content with URL".to_string();
-        let instructions = "URL instructions".to_string();
+        let context = "URL context".to_string();
         let category = "URL category".to_string();
         let user_id = "user123".to_string();
         let title = "page_title".to_string();
@@ -115,7 +115,7 @@ mod tests {
 
         let text_content = TextContent::new(
             text.clone(),
-            instructions.clone(),
+            Some(context.clone()),
             category.clone(),
             None,
             url_info.clone(),
@@ -137,13 +137,13 @@ mod tests {
 
         // Create initial text content
         let initial_text = "Initial text".to_string();
-        let initial_instructions = "Initial instructions".to_string();
+        let initial_context = "Initial context".to_string();
         let initial_category = "Initial category".to_string();
         let user_id = "user123".to_string();
 
         let text_content = TextContent::new(
             initial_text,
-            initial_instructions,
+            Some(initial_context),
             initial_category,
             None,
             None,
@@ -158,20 +158,14 @@ mod tests {
         assert!(stored.is_some());
 
         // New values for patch
-        let new_instructions = "Updated instructions";
+        let new_context = "Updated context";
         let new_category = "Updated category";
         let new_text = "Updated text content";
 
         // Apply the patch
-        TextContent::patch(
-            &text_content.id,
-            new_instructions,
-            new_category,
-            new_text,
-            &db,
-        )
-        .await
-        .expect("Failed to patch text content");
+        TextContent::patch(&text_content.id, new_context, new_category, new_text, &db)
+            .await
+            .expect("Failed to patch text content");
 
         // Retrieve the updated content
         let updated: Option<TextContent> = db
@@ -183,7 +177,7 @@ mod tests {
         let updated_content = updated.unwrap();
 
         // Verify the updates
-        assert_eq!(updated_content.instructions, new_instructions);
+        assert_eq!(updated_content.context, Some(new_context.to_string()));
         assert_eq!(updated_content.category, new_category);
         assert_eq!(updated_content.text, new_text);
         assert!(updated_content.updated_at > text_content.updated_at);