starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-04-17 22:49:43 +02:00
Code Issues Packages Projects Releases 10 Wiki Activity

refactored queue into Job

Browse Source
This commit is contained in:
Per Stark
2025-03-05 16:14:18 +01:00
parent cada21dbf6
commit 7d79f468aa
29 changed files with 401 additions and 490 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/main/src/server.rs
View File

@@ -20,8 +20,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Set up router states
let html_state = HtmlState::new(&config).await?;
let api_state = ApiState {
surreal_db_client: html_state.surreal_db_client.clone(),
job_queue: html_state.job_queue.clone(),
db: html_state.db.clone(),
};
// Create Axum router

58
crates/main/src/worker.rs
View File

@@ -1,12 +1,9 @@
use std::sync::Arc;
use common::{
ingress::{
content_processor::ContentProcessor,
jobqueue::{JobQueue, MAX_ATTEMPTS},
},
ingress::content_processor::ContentProcessor,
storage::{
db::{get_item, SurrealDbClient},
db::SurrealDbClient,
types::job::{Job, JobStatus},
},
utils::config::get_config,
@@ -40,27 +37,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let openai_client = Arc::new(async_openai::Client::new());
let job_queue = JobQueue::new(surreal_db_client.clone());
let content_processor = ContentProcessor::new(surreal_db_client, openai_client.clone()).await?;
let content_processor =
ContentProcessor::new(surreal_db_client.clone(), openai_client.clone()).await?;
loop {
// First, check for any unfinished jobs
let unfinished_jobs = job_queue.get_unfinished_jobs().await?;
let unfinished_jobs = Job::get_unfinished_jobs(&surreal_db_client).await?;
if !unfinished_jobs.is_empty() {
info!("Found {} unfinished jobs", unfinished_jobs.len());
for job in unfinished_jobs {
job_queue
.process_job(job, &content_processor, openai_client.clone())
.await?;
content_processor.process_job(job).await?;
}
}
// If no unfinished jobs, start listening for new ones
info!("Listening for new jobs...");
let mut job_stream = job_queue.listen_for_jobs().await?;
let mut job_stream = Job::listen_for_jobs(&surreal_db_client).await?;
while let Some(notification) = job_stream.next().await {
match notification {
@@ -69,14 +63,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
match notification.action {
Action::Create => {
if let Err(e) = job_queue
.process_job(
notification.data,
&content_processor,
openai_client.clone(),
)
.await
{
if let Err(e) = content_processor.process_job(notification.data).await {
error!("Error processing job: {}", e);
}
}
@@ -93,20 +80,18 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
JobStatus::InProgress { attempts, .. } => {
// Only process if this is a retry after an error, not our own update
if let Ok(Some(current_job)) =
get_item::<Job>(&job_queue.db.client, &notification.data.id)
.await
if let Ok(Some(current_job)) = surreal_db_client
.get_item::<Job>(&notification.data.id)
.await
{
match current_job.status {
JobStatus::Error(_) if attempts < MAX_ATTEMPTS => {
JobStatus::Error(_)
if attempts
< common::storage::types::job::MAX_ATTEMPTS =>
{
// This is a retry after an error
if let Err(e) = job_queue
.process_job(
current_job,
&content_processor,
openai_client.clone(),
)
.await
if let Err(e) =
content_processor.process_job(current_job).await
{
error!("Error processing job retry: {}", e);
}
@@ -123,13 +108,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}
JobStatus::Created => {
// Shouldn't happen with Update action, but process if it does
if let Err(e) = job_queue
.process_job(
notification.data,
&content_processor,
openai_client.clone(),
)
.await
if let Err(e) =
content_processor.process_job(notification.data).await
{
error!("Error processing job: {}", e);
}