fix: name harmonization of endpoints & ingestion security hardening

html-router/src/lib.rs

@@ -35,7 +35,9 @@ where
         .add_protected_routes(routes::chat::router())
         .add_protected_routes(routes::content::router())
         .add_protected_routes(routes::knowledge::router())
-        .add_protected_routes(routes::ingestion::router())
+        .add_protected_routes(routes::ingestion::router(
+            app_state.config.ingest_max_body_bytes,
+        ))
         .add_protected_routes(routes::scratchpad::router())
         .with_compression()
         .build()

html-router/src/routes/ingestion/handlers.rs

@@ -2,9 +2,10 @@ use std::{pin::Pin, time::Duration};
 
 use axum::{
     extract::{Query, State},
+    http::StatusCode,
     response::{
         sse::{Event, KeepAlive},
-        Html, IntoResponse, Sse,
+        Html, IntoResponse, Response, Sse,
     },
 };
 use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
@@ -23,6 +24,7 @@ use common::{
         ingestion_task::{IngestionTask, TaskState},
         user::User,
     },
+    utils::ingest_limits::{validate_ingest_input, IngestValidationError},
 };
 
 use crate::{
@@ -34,30 +36,32 @@ use crate::{
     AuthSessionType,
 };
 
-pub async fn show_ingress_form(
+pub async fn show_ingest_form(
     State(state): State<HtmlState>,
     RequireUser(user): RequireUser,
 ) -> Result<impl IntoResponse, HtmlError> {
     let user_categories = User::get_user_categories(&user.id, &state.db).await?;
 
     #[derive(Serialize)]
-    pub struct ShowIngressFormData {
+    pub struct ShowIngestFormData {
         user_categories: Vec<String>,
     }
 
     Ok(TemplateResponse::new_template(
         "ingestion_modal.html",
-        ShowIngressFormData { user_categories },
+        ShowIngestFormData { user_categories },
     ))
 }
 
-pub async fn hide_ingress_form(
+pub async fn hide_ingest_form(
     RequireUser(_user): RequireUser,
 ) -> Result<impl IntoResponse, HtmlError> {
-    Ok(Html(
-        "<a class='btn btn-primary' hx-get='/ingress-form' hx-swap='outerHTML'>Add Content</a>",
+    Ok(
+        Html(
+            "<a class='btn btn-primary' hx-get='/ingest-form' hx-swap='outerHTML'>Add Content</a>",
+        )
+        .into_response(),
     )
-    .into_response())
 }
 
 #[derive(Debug, TryFromMultipart)]
@@ -65,34 +69,22 @@ pub struct IngestionParams {
     pub content: Option<String>,
     pub context: String,
     pub category: String,
-    #[form_data(limit = "10000000")] // Adjust limit as needed
+    #[form_data(limit = "20000000")]
     #[form_data(default)]
     pub files: Vec<FieldData<NamedTempFile>>,
 }
 
-pub async fn process_ingress_form(
+pub async fn process_ingest_form(
     State(state): State<HtmlState>,
     RequireUser(user): RequireUser,
     TypedMultipart(input): TypedMultipart<IngestionParams>,
-) -> Result<impl IntoResponse, HtmlError> {
-    #[derive(Serialize)]
-    pub struct IngressFormData {
-        context: String,
-        content: String,
-        category: String,
-        error: String,
-    }
-
+) -> Result<Response, HtmlError> {
     if input.content.as_ref().is_none_or(|c| c.len() < 2) && input.files.is_empty() {
-        return Ok(TemplateResponse::new_template(
-            "index/signed_in/ingress_form.html",
-            IngressFormData {
-                context: input.context.clone(),
-                content: input.content.clone().unwrap_or_default(),
-                category: input.category.clone(),
-                error: "You need to either add files or content".to_string(),
-            },
-        ));
+        return Ok((
+            StatusCode::BAD_REQUEST,
+            "You need to either add files or content",
+        )
+            .into_response());
     }
 
     let content_bytes = input.content.as_ref().map_or(0, |c| c.len());
@@ -101,6 +93,22 @@ pub async fn process_ingress_form(
     let category_bytes = input.category.len();
     let file_count = input.files.len();
 
+    match validate_ingest_input(
+        &state.config,
+        input.content.as_deref(),
+        &input.context,
+        &input.category,
+        file_count,
+    ) {
+        Ok(()) => {}
+        Err(IngestValidationError::PayloadTooLarge(message)) => {
+            return Ok((StatusCode::PAYLOAD_TOO_LARGE, message).into_response());
+        }
+        Err(IngestValidationError::BadRequest(message)) => {
+            return Ok((StatusCode::BAD_REQUEST, message).into_response());
+        }
+    }
+
     info!(
         user_id = %user.id,
         has_content,
@@ -108,7 +116,7 @@ pub async fn process_ingress_form(
         context_bytes,
         category_bytes,
         file_count,
-        "Received ingestion form submission"
+        "Received ingest form submission"
     );
 
     let file_infos = try_join_all(input.files.into_iter().map(|file| {
@@ -137,10 +145,10 @@ pub async fn process_ingress_form(
         tasks: Vec<IngestionTask>,
     }
 
-    Ok(TemplateResponse::new_template(
-        "dashboard/current_task.html",
-        NewTasksData { tasks },
-    ))
+    Ok(
+        TemplateResponse::new_template("dashboard/current_task.html", NewTasksData { tasks })
+            .into_response(),
+    )
 }
 
 #[derive(Deserialize)]

html-router/src/routes/ingestion/mod.rs

@@ -1,22 +1,22 @@
 mod handlers;
 
-use axum::{extract::FromRef, routing::get, Router};
-use handlers::{
-    get_task_updates_stream, hide_ingress_form, process_ingress_form, show_ingress_form,
-};
+use axum::{extract::DefaultBodyLimit, extract::FromRef, routing::get, Router};
+use handlers::{get_task_updates_stream, hide_ingest_form, process_ingest_form, show_ingest_form};
 
 use crate::html_state::HtmlState;
 
-pub fn router<S>() -> Router<S>
+pub fn router<S>(max_body_bytes: usize) -> Router<S>
 where
     S: Clone + Send + Sync + 'static,
     HtmlState: FromRef<S>,
 {
     Router::new()
         .route(
-            "/ingress-form",
-            get(show_ingress_form).post(process_ingress_form),
+            "/ingest-form",
+            get(show_ingest_form)
+                .post(process_ingest_form)
+                .layer(DefaultBodyLimit::max(max_body_bytes)),
         )
         .route("/task/status-stream", get(get_task_updates_stream))
-        .route("/hide-ingress-form", get(hide_ingress_form))
+        .route("/hide-ingest-form", get(hide_ingest_form))
 }

html-router/templates/dashboard/_base.html

@@ -2,7 +2,7 @@
 
 {% block dashboard_header %}
   <h1 class="text-xl font-extrabold tracking-tight">Dashboard</h1>
-  <button class="nb-btn nb-cta" hx-get="/ingress-form" hx-target="#modal" hx-swap="innerHTML">
+  <button class="nb-btn nb-cta" hx-get="/ingest-form" hx-target="#modal" hx-swap="innerHTML">
     {% include "icons/send_icon.html" %}
     <span class="ml-2">Add Content</span>
   </button>

html-router/templates/ingestion_modal.html

@@ -3,7 +3,7 @@
 {% block modal_class %}max-w-3xl{% endblock %}
 
 {% block form_attributes %}
-hx-post="/ingress-form"
+hx-post="/ingest-form"
 enctype="multipart/form-data"
 {% endblock %}
 

html-router/templates/sidebar.html

@@ -18,7 +18,7 @@
   </li>
   {% endfor %}
   <li>
-    <button class="nb-btn nb-cta w-full flex items-center gap-3 justify-start mt-2" hx-get="/ingress-form"
+    <button class="nb-btn nb-cta w-full flex items-center gap-3 justify-start mt-2" hx-get="/ingest-form"
       hx-target="#modal" hx-swap="innerHTML">{% include "icons/send_icon.html" %} Add
       Content</button>
   </li>