feat: job queue html

This commit is contained in:
Per Stark committed 2025-01-10 20:37:16 +01:00
1 parent d77f07c626
commit 8a7ecd635f
6 files changed
+155 -62

No files matched your search

+1 -1
View File
@@ -1 +1 @@
*,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }/*! tailwindcss v3.4.17 | MIT License | https://tailwindcss.com*/*,:after,:before{border:0 solid #e5e7eb;box-sizing:border-box}:after,:before{--tw-content:""}:host,html{line-height:1.5;-webkit-text-size-adjust:100%;font-family:ui-sans-serif,system-ui,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-tap-highlight-color:transparent}body{line-height:inherit;margin:0}hr{border-top-width:1px;color:inherit;height:0}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-feature-settings:normal;font-size:1em;font-variation-settings:normal}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{border-collapse:collapse;border-color:inherit;text-indent:0}button,input,optgroup,select,textarea{color:inherit;font-family:inherit;font-feature-settings:inherit;font-size:100%;font-variation-settings:inherit;font-weight:inherit;letter-spacing:inherit;line-height:inherit;margin:0;padding:0}button,select{text-transform:none}button,input:where([type=button]),input:where([type=reset]),input:where([type=submit]){-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{color:#9ca3af;opacity:1}input::placeholder,textarea::placeholder{color:#9ca3af;opacity:1}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{height:auto;max-width:100%}[hidden]:where(:not([hidden=until-found])){display:none}:root,[data-theme]{background-color:var(--fallback-b1,oklch(var(--b1)/1));color:var(--fallback-bc,oklch(var(--bc)/1))}@supports not (color:oklch(0% 0 0)){:root{color-scheme:light;--fallback-p:#491eff;--fallback-pc:#d4dbff;--fallback-s:#ff41c7;--Line truncated
*,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }/*! tailwindcss v3.4.17 | MIT License | https://tailwindcss.com*/*,:after,:before{border:0 solid #e5e7eb;box-sizing:border-box}:after,:before{--tw-content:""}:host,html{line-height:1.5;-webkit-text-size-adjust:100%;font-family:ui-sans-serif,system-ui,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-tap-highlight-color:transparent}body{line-height:inherit;margin:0}hr{border-top-width:1px;color:inherit;height:0}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-feature-settings:normal;font-size:1em;font-variation-settings:normal}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{border-collapse:collapse;border-color:inherit;text-indent:0}button,input,optgroup,select,textarea{color:inherit;font-family:inherit;font-feature-settings:inherit;font-size:100%;font-variation-settings:inherit;font-weight:inherit;letter-spacing:inherit;line-height:inherit;margin:0;padding:0}button,select{text-transform:none}button,input:where([type=button]),input:where([type=reset]),input:where([type=submit]){-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{color:#9ca3af;opacity:1}input::placeholder,textarea::placeholder{color:#9ca3af;opacity:1}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{height:auto;max-width:100%}[hidden]:where(:not([hidden=until-found])){display:none}:root,[data-theme]{background-color:var(--fallback-b1,oklch(var(--b1)/1));color:var(--fallback-bc,oklch(var(--bc)/1))}@supports not (color:oklch(0% 0 0)){:root{color-scheme:light;--fallback-p:#491eff;--fallback-pc:#d4dbff;--fallback-s:#ff41c7;--Line truncated
+1 -1
View File
@@ -158,7 +158,7 @@ fn html_routes(
get(show_ingress_form).post(process_ingress_form),
)
.route("/queue", get(show_queue_tasks))
.route("/queue/:delivery_tag", post(delete_task))
.route("/queue/:delivery_tag", delete(delete_task))
.route("/account", get(show_account_page))
.route("/set-api-key", post(set_api_key))
.route("/delete-account", delete(delete_account))
+69 -8
View File
@@ -1,11 +1,18 @@
use std::sync::Arc;
use futures::StreamExt;
use surrealdb::Action;
use tracing::{error, info};
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
use zettle_db::{
ingress::{content_processor::ContentProcessor, jobqueue::JobQueue},
storage::db::SurrealDbClient,
ingress::{
content_processor::ContentProcessor,
jobqueue::{self, JobQueue, MAX_ATTEMPTS},
},
storage::{
db::{get_item, SurrealDbClient},
types::job::{Job, JobStatus},
},
utils::config::get_config,
};
@@ -54,12 +61,66 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
while let Some(notification) = job_stream.next().await {
match notification {
Ok(notification) => {
info!("Received new job: {}", notification.data.id);
if let Err(e) = job_queue
.process_job(notification.data, &content_processor)
.await
{
error!("Error processing job: {}", e);
info!("Received notification: {:?}", notification);
match notification.action {
Action::Create => {
if let Err(e) = job_queue
.process_job(notification.data, &content_processor)
.await
{
error!("Error processing job: {}", e);
}
}
Action::Update => {
match notification.data.status {
JobStatus::Completed
| JobStatus::Error(_)
| JobStatus::Cancelled => {
info!(
"Skipping already completed/error/cancelled job: {}",
notification.data.id
);
continue;
}
JobStatus::InProgress { attempts, .. } => {
// Only process if this is a retry after an error, not our own update
if let Ok(Some(current_job)) =
get_item::<Job>(&job_queue.db.client, &notification.data.id)
.await
{
match current_job.status {
JobStatus::Error(_) if attempts < MAX_ATTEMPTS => {
// This is a retry after an error
if let Err(e) = job_queue
.process_job(current_job, &content_processor)
.await
{
error!("Error processing job retry: {}", e);
}
}
_ => {
info!(
"Skipping in-progress update for job: {}",
notification.data.id
);
continue;
}
}
}
}
JobStatus::Created => {
// Shouldn't happen with Update action, but process if it does
if let Err(e) = job_queue
.process_job(notification.data, &content_processor)
.await
{
error!("Error processing job: {}", e);
}
}
}
}
_ => {} // Ignore other actions
}
}
Err(e) => error!("Error in job notification: {}", e),
+25 -27
View File
@@ -9,7 +9,7 @@ use tracing::{error, info};
use crate::{
error::AppError,
storage::{
db::{store_item, SurrealDbClient},
db::{delete_item, get_item, store_item, SurrealDbClient},
types::{
job::{Job, JobStatus},
StoredObject,
@@ -20,10 +20,10 @@ use crate::{
use super::{content_processor::ContentProcessor, types::ingress_object::IngressObject};
pub struct JobQueue {
db: Arc<SurrealDbClient>,
pub db: Arc<SurrealDbClient>,
}
const MAX_ATTEMPTS: u32 = 3;
pub const MAX_ATTEMPTS: u32 = 3;
impl JobQueue {
pub fn new(db: Arc<SurrealDbClient>) -> Self {
@@ -50,34 +50,22 @@ impl JobQueue {
}
pub async fn delete_job(&self, id: &str, user_id: &str) -> Result<(), AppError> {
// First, validate that the job exists and belongs to the user
let job: Option<Job> = self
.db
.query("SELECT * FROM job WHERE id = $id AND user_id = $user_id")
.bind(("id", id.to_string()))
.bind(("user_id", user_id.to_string()))
get_item::<Job>(&self.db.client, id)
.await?
.take(0)?;
// If no job is found or it doesn't belong to the user, return Unauthorized
if job.is_none() {
error!("Unauthorized attempt to delete job {id} by user {user_id}");
return Err(AppError::Auth("Not authorized to delete this job".into()));
}
.filter(|job| job.user_id == user_id)
.ok_or_else(|| {
error!("Unauthorized attempt to delete job {id} by user {user_id}");
AppError::Auth("Not authorized to delete this job".into())
})?;
info!("Deleting job {id} for user {user_id}");
// If validation passes, delete the job
let _deleted: Option<Job> = self
.db
.delete((Job::table_name(), id))
delete_item::<Job>(&self.db.client, id)
.await
.map_err(AppError::Database)?;
Ok(())
}
/// Update status for job
pub async fn update_status(
&self,
id: &str,
@@ -89,13 +77,10 @@ impl JobQueue {
.as_millis()
.to_string();
let status_value =
serde_json::to_value(status).map_err(|e| AppError::LLMParsing(e.to_string()))?;
let job: Option<Job> = self
.db
.update((Job::table_name(), id))
.patch(PatchOp::replace("/status", status_value))
.patch(PatchOp::replace("/status", status))
.patch(PatchOp::replace("/updated_at", now))
.await?;
@@ -109,14 +94,27 @@ impl JobQueue {
self.db.select("job").live().await
}
/// Get unfinished jobs, ie newly created and in progress up two times
pub async fn get_unfinished_jobs(&self) -> Result<Vec<Job>, AppError> {
let jobs: Vec<Job> = self
.db
.query(
"SELECT * FROM job WHERE status.Created = true OR (status.InProgress.attempts < $max_attempts) ORDER BY created_at ASC")
"SELECT * FROM type::table($table)
WHERE
status = 'Created'
OR (
status.InProgress != NONE
AND status.InProgress.attempts < $max_attempts
)
ORDER BY created_at ASC",
)
.bind(("table", Job::table_name()))
.bind(("max_attempts", MAX_ATTEMPTS))
.await?
.take(0)?;
println!("Unfinished jobs found: {}", jobs.len());
Ok(jobs)
}
+14 -3
View File
@@ -2,7 +2,10 @@ use crate::{
error::{AppError, HtmlError},
page_data,
server::AppState,
storage::types::{job::Job, user::User},
storage::types::{
job::{Job, JobStatus},
user::User,
},
};
use axum::{
extract::{Path, State},
@@ -11,10 +14,11 @@ use axum::{
use axum_session_auth::AuthSession;
use axum_session_surreal::SessionSurrealPool;
use surrealdb::{engine::any::Any, Surreal};
use tracing::info;
use super::render_template;
page_data!(ShowQueueTasks, "queue_tasks.html", {jobs: Vec<Job>});
page_data!(ShowQueueTasks, "queue_tasks.html", {user : User,jobs: Vec<Job>});
pub async fn show_queue_tasks(
State(state): State<AppState>,
@@ -31,9 +35,16 @@ pub async fn show_queue_tasks(
.await
.map_err(|e| HtmlError::new(e, state.templates.clone()))?;
for job in &jobs {
match job.status {
JobStatus::Created => info!("Found a created job"),
_ => continue,
}
}
let rendered = render_template(
ShowQueueTasks::template_name(),
ShowQueueTasks { jobs },
ShowQueueTasks { jobs, user },
state.templates.clone(),
)
.map_err(|e| HtmlError::new(AppError::from(e), state.templates.clone()))?;
+45 -22
View File
@@ -1,37 +1,60 @@
{% extends "body_base.html" %}
{% block main %}
<div class="container mx-auto p-4">
<h1 class="text-2xl font-bold mb-4">Queue Tasks</h1>
{% if tasks|length == 0 %}
<h1 class="text-2xl font-bold mb-4">Active Tasks</h1>
{% if not jobs %}
<div class="alert alert-info">
<span>No tasks in queue</span>
<span>No active tasks</span>
</div>
{% else %}
<div class="grid gap-4">
{% for (task, tag) in tasks %}
<div class="card bg-base-200 shadow-xl">
{% for job in jobs %}
{% if job.status == "Created" or job.status is mapping and job.status.InProgress %}
<div class="card bg-base-200 shadow-xl" id="job-card-{{ job.id }}">
<div class="card-body">
{% if task is object("Url") %}
<h2 class="card-title">URL Task</h2>
<p>URL: {{ task.url }}</p>
{% elif task is object("Text") %}
<h2 class="card-title">Text Task</h2>
<p>Text: {{ task.text }}</p>
{% elif task is object("File") %}
<h2 class="card-title">File Task</h2>
<p>File: {{ task.file_info.original_name }}</p>
{% endif %}
<p>Instructions: {{ task.instructions }}</p>
<p>Category: {{ task.category }}</p>
<div class="card-actions justify-end">
<form action="/queue/{{ tag }}" method="POST">
<button class="btn btn-error">Delete Task</button>
<div class="card-title">
{% if job.content.Url %}
<h2>URL Task</h2>
<p class="text-sm text-gray-500 break-all">{{ job.content.Url.url }}</p>
{% elif job.content.File %}
<h2>File Task</h2>
<p class="text-sm text-gray-500">{{ job.content.File.file_info.path }}</p>
{% elif job.content.Text %}
<h2>Text Task</h2>
<p class="text-sm text-gray-500">{{ job.content.Text.text }}</p>
{% endif %}
</div>
<div class="space-y-2">
<p><span class="font-medium">Status:</span>
{% if job.status == "Created" %}
Created
{% elif job.status.InProgress %}
In Progress
{% endif %}
</p>
{% if job.status.InProgress %}
<p><span class="font-medium">Attempts:</span> {{ job.status.InProgress.attempts }}</p>
<p><span class="font-medium">Last Attempt:</span>
{{ job.status.InProgress.last_attempt }}</p>
{% endif %}
<p><span class="font-medium">Category:</span>
{% if job.content.Url %}
{{ job.content.Url.category }}
{% elif job.content.File %}
{{ job.content.File.category }}
{% elif job.content.Text %}
{{ job.content.Text.category }}
{% endif %}
</p>
</div>
<div class="card-actions justify-end mt-4">
<form hx-delete="/queue/{{ job.id }}" hx-target="#job-card-{{ job.id }}" hx-swap="outerHTML">
<button class="btn btn-error">Cancel Task</button>
</form>
</div>
</div>
</div>
{% endif %}
{% endfor %}
</div>
{% endif %}