in progress, routers and main split up

crates/main/Cargo.toml

@@ -0,0 +1,58 @@
+[package]
+name = "main"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tokio = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+anyhow = { workspace = true }
+tracing = { workspace = true }
+axum = { workspace = true }
+
+async-openai = "0.24.1"
+async-stream = "0.3.6"
+axum-htmx = "0.6.0"
+axum_session = "0.14.4"
+axum_session_auth = "0.14.1"
+axum_session_surreal = "0.2.1"
+axum_typed_multipart = "0.12.1"
+chrono = { version = "0.4.39", features = ["serde"] }
+chrono-tz = "0.10.1"
+config = "0.15.4"
+futures = "0.3.31"
+json-stream-parser = "0.1.4"
+lettre = { version = "0.11.11", features = ["rustls-tls"] }
+mime = "0.3.17"
+mime_guess = "2.0.5"
+minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
+minijinja-autoreload = "2.5.0"
+minijinja-contrib = { version = "2.6.0", features = ["datetime", "timezone"] }
+mockall = "0.13.0"
+plotly = "0.12.1"
+reqwest = {version = "0.12.12", features = ["charset", "json"]}
+scraper = "0.22.0"
+sha2 = "0.10.8"
+surrealdb = "2.0.4"
+tempfile = "3.12.0"
+text-splitter = "0.18.1"
+tiktoken-rs = "0.6.0"
+tower-http = { version = "0.6.2", features = ["fs"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+url = { version = "2.5.2", features = ["serde"] }
+uuid = { version = "1.10.0", features = ["v4", "serde"] }
+
+# Reference to api-router
+api-router = { path = "../api-router" }
+html-router = { path = "../html-router" }
+common = { path = "../common" }
+
+[[bin]]
+name = "server"
+path = "src/server.rs"
+
+[[bin]]
+name = "worker"
+path = "src/worker.rs"

crates/main/src/server.rs

@@ -0,0 +1,47 @@
+use api_router::{api_routes_v1, api_state::ApiState};
+use axum::{extract::FromRef, Router};
+use common::utils::config::get_config;
+use html_router::{html_routes, html_state::HtmlState};
+use tracing::info;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Set up tracing
+    tracing_subscriber::registry()
+        .with(fmt::layer())
+        .with(EnvFilter::from_default_env())
+        .try_init()
+        .ok();
+
+    // Get config
+    let config = get_config()?;
+
+    // Set up router states
+    let html_state = HtmlState::new(&config).await?;
+    let api_state = ApiState {
+        surreal_db_client: html_state.surreal_db_client.clone(),
+        job_queue: html_state.job_queue.clone(),
+    };
+
+    // Create Axum router
+    let app = Router::new()
+        .nest("/api/v1", api_routes_v1(&api_state))
+        .nest("/", html_routes(&html_state))
+        .with_state(AppState {
+            api_state,
+            html_state,
+        });
+
+    info!("Listening on 0.0.0.0:3000");
+    let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
+    axum::serve(listener, app).await?;
+
+    Ok(())
+}
+
+#[derive(Clone, FromRef)]
+struct AppState {
+    api_state: ApiState,
+    html_state: HtmlState,
+}

crates/main/src/worker.rs

@@ -0,0 +1,150 @@
+use std::sync::Arc;
+
+use common::{
+    ingress::{
+        content_processor::ContentProcessor,
+        jobqueue::{JobQueue, MAX_ATTEMPTS},
+    },
+    storage::{
+        db::{get_item, SurrealDbClient},
+        types::job::{Job, JobStatus},
+    },
+    utils::config::get_config,
+};
+use futures::StreamExt;
+use surrealdb::Action;
+use tracing::{error, info};
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Set up tracing
+    tracing_subscriber::registry()
+        .with(fmt::layer())
+        .with(EnvFilter::from_default_env())
+        .try_init()
+        .ok();
+
+    let config = get_config()?;
+
+    let surreal_db_client = Arc::new(
+        SurrealDbClient::new(
+            &config.surrealdb_address,
+            &config.surrealdb_username,
+            &config.surrealdb_password,
+            &config.surrealdb_namespace,
+            &config.surrealdb_database,
+        )
+        .await?,
+    );
+
+    let openai_client = Arc::new(async_openai::Client::new());
+
+    let job_queue = JobQueue::new(surreal_db_client.clone());
+
+    let content_processor = ContentProcessor::new(surreal_db_client, openai_client.clone()).await?;
+
+    loop {
+        // First, check for any unfinished jobs
+        let unfinished_jobs = job_queue.get_unfinished_jobs().await?;
+
+        if !unfinished_jobs.is_empty() {
+            info!("Found {} unfinished jobs", unfinished_jobs.len());
+
+            for job in unfinished_jobs {
+                job_queue
+                    .process_job(job, &content_processor, openai_client.clone())
+                    .await?;
+            }
+        }
+
+        // If no unfinished jobs, start listening for new ones
+        info!("Listening for new jobs...");
+        let mut job_stream = job_queue.listen_for_jobs().await?;
+
+        while let Some(notification) = job_stream.next().await {
+            match notification {
+                Ok(notification) => {
+                    info!("Received notification: {:?}", notification);
+
+                    match notification.action {
+                        Action::Create => {
+                            if let Err(e) = job_queue
+                                .process_job(
+                                    notification.data,
+                                    &content_processor,
+                                    openai_client.clone(),
+                                )
+                                .await
+                            {
+                                error!("Error processing job: {}", e);
+                            }
+                        }
+                        Action::Update => {
+                            match notification.data.status {
+                                JobStatus::Completed
+                                | JobStatus::Error(_)
+                                | JobStatus::Cancelled => {
+                                    info!(
+                                        "Skipping already completed/error/cancelled job: {}",
+                                        notification.data.id
+                                    );
+                                    continue;
+                                }
+                                JobStatus::InProgress { attempts, .. } => {
+                                    // Only process if this is a retry after an error, not our own update
+                                    if let Ok(Some(current_job)) =
+                                        get_item::<Job>(&job_queue.db.client, &notification.data.id)
+                                            .await
+                                    {
+                                        match current_job.status {
+                                            JobStatus::Error(_) if attempts < MAX_ATTEMPTS => {
+                                                // This is a retry after an error
+                                                if let Err(e) = job_queue
+                                                    .process_job(
+                                                        current_job,
+                                                        &content_processor,
+                                                        openai_client.clone(),
+                                                    )
+                                                    .await
+                                                {
+                                                    error!("Error processing job retry: {}", e);
+                                                }
+                                            }
+                                            _ => {
+                                                info!(
+                                                    "Skipping in-progress update for job: {}",
+                                                    notification.data.id
+                                                );
+                                                continue;
+                                            }
+                                        }
+                                    }
+                                }
+                                JobStatus::Created => {
+                                    // Shouldn't happen with Update action, but process if it does
+                                    if let Err(e) = job_queue
+                                        .process_job(
+                                            notification.data,
+                                            &content_processor,
+                                            openai_client.clone(),
+                                        )
+                                        .await
+                                    {
+                                        error!("Error processing job: {}", e);
+                                    }
+                                }
+                            }
+                        }
+                        _ => {} // Ignore other actions
+                    }
+                }
+                Err(e) => error!("Error in job notification: {}", e),
+            }
+        }
+
+        // If we reach here, the stream has ended (connection lost?)
+        error!("Job stream ended unexpectedly, reconnecting...");
+        tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+    }
+}