moved entry points to bin to clarify

This commit is contained in:
Per Stark
2024-11-22 12:33:47 +01:00
parent 5f64fbd3fb
commit b5ad101d3f
6 changed files with 12 additions and 6 deletions

View File

@@ -1,3 +1,5 @@
use std::time::Instant;
use text_splitter::TextSplitter;
use tracing::{debug, info};
@@ -35,11 +37,17 @@ impl ContentProcessor {
// Store original content
store_item(&self.db_client, content.clone()).await?;
let now = Instant::now();
// Process in parallel where possible
let (analysis, _similar_chunks) = tokio::try_join!(
self.perform_semantic_analysis(content),
self.find_similar_content(content),
)?;
let end = now.elapsed();
info!(
"{:?} time elapsed during creation of entities and relationships",
end
);
// Convert and store entities
let (entities, relationships) = analysis

View File

@@ -1,4 +1,4 @@
use futures_lite::stream::StreamExt;
use futures::StreamExt;
use lapin::{message::Delivery, options::*, types::FieldTable, Channel, Consumer, Queue};
use crate::{