starred/minne
1
0
Fork 0
mirror of https://github.com/perstarkse/minne.git synced 2026-03-26 19:31:32 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

multipart wip

Browse Source
This commit is contained in:
Per Stark
2024-09-24 14:02:38 +02:00
parent 990f995caf
commit eed07c31f9
15 changed files with 871 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/consumer.rs
View File

@@ -29,28 +29,29 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Consumer connected to RabbitMQ");
// Start consuming messages
loop {
match consumer.consume().await {
Ok((message, delivery)) => {
info!("Received message: {}", message);
// Process the message here
// For example, you could insert it into a database
// process_message(&message).await?;
consumer.process_messages().await?;
// loop {
// match consumer.consume().await {
// Ok((message, delivery)) => {
// info!("Received message: {}", message);
// // Process the message here
// // For example, you could insert it into a database
// // process_message(&message).await?;
info!("Done processing, acking");
consumer.ack_delivery(delivery).await?
}
Err(RabbitMQError::ConsumeError(e)) => {
error!("Error consuming message: {}", e);
// Optionally add a delay before trying again
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
Err(e) => {
error!("Unexpected error: {}", e);
break;
}
}
}
// info!("Done processing, acking");
// consumer.ack_delivery(delivery).await?
// }
// Err(RabbitMQError::ConsumeError(e)) => {
// error!("Error consuming message: {}", e);
// // Optionally add a delay before trying again
// tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
// }
// Err(e) => {
// error!("Unexpected error: {}", e);
// break;
// }
// }
// }
Ok(())
}

2
src/lib.rs
View File

@@ -1 +1,3 @@
pub mod models;
pub mod rabbitmq;
pub mod utils;

93
src/models/ingress.rs Normal file
View File

@@ -0,0 +1,93 @@
use axum::extract::Multipart;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::fs;
use tracing::info;
use url::Url;
use uuid::Uuid;
use sha2::{Digest, Sha256};
use std::path::Path;
use mime_guess::from_path;
use axum_typed_multipart::{FieldData, TryFromMultipart };
use tempfile::NamedTempFile;
#[derive(Debug, TryFromMultipart)]
pub struct IngressMultipart {
/// JSON content field
pub content: Option<String>,
pub instructions: String,
pub category: String,
/// Optional file
#[form_data(limit = "unlimited")]
pub file: Option<FieldData<NamedTempFile>>,
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct FileInfo {
pub uuid: Uuid,
pub sha256: String,
pub path: String,
pub mime_type: String,
}
#[derive(Debug, Deserialize, Serialize)]
pub enum Content {
Url(String),
Text(String),
}
#[derive(Serialize, Deserialize, Debug)]
pub struct IngressContent {
pub content: Option<Content>,
pub instructions: String,
pub category: String,
pub files: Option<Vec<FileInfo>>,
}
/// Error types for file and content handling.
#[derive(Error, Debug)]
pub enum IngressContentError {
#[error("IO error occurred: {0}")]
Io(#[from] std::io::Error),
#[error("UTF-8 conversion error: {0}")]
Utf8(#[from] std::string::FromUtf8Error),
#[error("MIME type detection failed for input: {0}")]
MimeDetection(String),
#[error("Unsupported MIME type: {0}")]
UnsupportedMime(String),
#[error("URL parse error: {0}")]
UrlParse(#[from] url::ParseError),
}
impl IngressContent {
/// Create a new `IngressContent` from `IngressMultipart`.
pub async fn new(
content: Option<String>, instructions: String, category: String,
file: Option<FileInfo>
) -> Result<IngressContent, IngressContentError> {
let content = if let Some(content_str) = content {
// Check if the content is a URL
if let Ok(url) = Url::parse(&content_str) {
info!("Detected URL: {}", url);
Some(Content::Url(url.to_string()))
} else {
info!("Treating input as plain text");
Some(Content::Text(content_str))
}
} else {
None
};
Ok(IngressContent {
content,
instructions,
category,
files: file.map(|f| vec![f]), // Single file wrapped in a Vec
})
}
}

1
src/models/mod.rs Normal file
View File

@@ -0,0 +1 @@
pub mod ingress;

52
src/rabbitmq/consumer.rs
View File

@@ -2,10 +2,13 @@ use lapin::{
message::Delivery, options::*, types::FieldTable, Channel, Consumer, Queue
};
use futures_lite::stream::StreamExt;
use tracing::info;
use crate::models::ingress::IngressContent;
use super::{RabbitMQCommon, RabbitMQConfig, RabbitMQError};
use tracing::{info, error};
use tokio::fs;
/// Struct to consume messages from RabbitMQ.
pub struct RabbitMQConsumer {
common: RabbitMQCommon,
pub queue: Queue,
@@ -26,7 +29,7 @@ impl RabbitMQConsumer {
// Initialize the consumer
let consumer = Self::initialize_consumer(&common.channel, &config).await?;
Ok(Self { common, queue, consumer})
Ok(Self { common, queue, consumer })
}
async fn initialize_consumer(channel: &Channel, config: &RabbitMQConfig) -> Result<Consumer, RabbitMQError> {
@@ -67,16 +70,21 @@ impl RabbitMQConsumer {
.map_err(|e| RabbitMQError::QueueError(e.to_string()))
}
pub async fn consume(&self) -> Result<(String, Delivery), RabbitMQError> {
/// Consumes a message and returns the deserialized IngressContent along with the Delivery
pub async fn consume(&self) -> Result<(IngressContent, Delivery), RabbitMQError> {
// Receive the next message
let delivery = self.consumer.clone().next().await
.ok_or_else(|| RabbitMQError::ConsumeError("No message received".to_string()))?
.map_err(|e| RabbitMQError::ConsumeError(e.to_string()))?;
let message = String::from_utf8_lossy(&delivery.data).to_string();
// Deserialize the message payload into IngressContent
let ingress: IngressContent = serde_json::from_slice(&delivery.data)
.map_err(|e| RabbitMQError::ConsumeError(format!("Deserialization Error: {}", e)))?;
Ok((message, delivery))
Ok((ingress, delivery))
}
/// Acknowledges the message after processing
pub async fn ack_delivery(&self, delivery: Delivery) -> Result<(), RabbitMQError> {
self.common.channel
.basic_ack(delivery.delivery_tag, BasicAckOptions::default())
@@ -85,5 +93,37 @@ impl RabbitMQConsumer {
Ok(())
}
}
/// Processes messages in a loop
pub async fn process_messages(&self) -> Result<(), RabbitMQError> {
loop {
match self.consume().await {
Ok((ingress, delivery)) => {
info!("Received ingress object: {:?}", ingress);
// Process the ingress object
self.handle_ingress_content(&ingress).await;
info!("Processing done, acknowledging message");
self.ack_delivery(delivery).await?;
}
Err(RabbitMQError::ConsumeError(e)) => {
error!("Error consuming message: {}", e);
// Optionally add a delay before trying again
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
Err(e) => {
error!("Unexpected error: {}", e);
break;
}
}
}
Ok(())
}
/// Handles the IngressContent based on its type
async fn handle_ingress_content(&self, ingress: &IngressContent) {
info!("Processing content: {:?}", ingress);
}
}

4
src/rabbitmq/mod.rs
View File

@@ -1,8 +1,8 @@
pub mod producer;
pub mod publisher;
pub mod consumer;
use lapin::{
options::{ExchangeDeclareOptions, QueueDeclareOptions}, types::FieldTable, Channel, Connection, ConnectionProperties, ExchangeKind, Queue
options::ExchangeDeclareOptions, types::FieldTable, Channel, Connection, ConnectionProperties, ExchangeKind
};
use thiserror::Error;

39
src/rabbitmq/producer.rs
View File

@@ -1,39 +0,0 @@
use lapin::{
options::*, publisher_confirm::Confirmation, BasicProperties,
};
use super::{RabbitMQCommon, RabbitMQConfig, RabbitMQError};
pub struct RabbitMQProducer {
common: RabbitMQCommon,
exchange_name: String,
routing_key: String,
}
impl RabbitMQProducer {
pub async fn new(config: &RabbitMQConfig) -> Result<Self, RabbitMQError> {
let common = RabbitMQCommon::new(config).await?;
common.declare_exchange(config, false).await?;
Ok(Self {
common,
exchange_name: config.exchange.clone(),
routing_key: config.routing_key.clone(),
})
}
pub async fn publish(&self, payload: &[u8]) -> Result<Confirmation, RabbitMQError> {
self.common.channel
.basic_publish(
&self.exchange_name,
&self.routing_key,
BasicPublishOptions::default(),
payload,
BasicProperties::default(),
)
.await
.map_err(|e| RabbitMQError::PublishError(e.to_string()))?
.await
.map_err(|e| RabbitMQError::PublishError(e.to_string()))
}
}

60
src/rabbitmq/publisher.rs Normal file
View File

@@ -0,0 +1,60 @@
use lapin::{
options::*, publisher_confirm::Confirmation, BasicProperties,
};
use crate::models::ingress::IngressContent;
use super::{RabbitMQCommon, RabbitMQConfig, RabbitMQError};
use tracing::{info, error};
pub struct RabbitMQProducer {
common: RabbitMQCommon,
exchange_name: String,
routing_key: String,
}
impl RabbitMQProducer {
pub async fn new(config: &RabbitMQConfig) -> Result<Self, RabbitMQError> {
let common = RabbitMQCommon::new(config).await?;
common.declare_exchange(config, false).await?;
Ok(Self {
common,
exchange_name: config.exchange.clone(),
routing_key: config.routing_key.clone(),
})
}
/// Publishes an IngressContent object to RabbitMQ after serializing it to JSON.
pub async fn publish(&self, ingress: &IngressContent) -> Result<Confirmation, RabbitMQError> {
// Serialize IngressContent to JSON
let payload = serde_json::to_vec(ingress)
.map_err(|e| {
error!("Serialization Error: {}", e);
RabbitMQError::PublishError(format!("Serialization Error: {}", e))
})?;
// Publish the serialized payload to RabbitMQ
let confirmation = self.common.channel
.basic_publish(
&self.exchange_name,
&self.routing_key,
BasicPublishOptions::default(),
&payload,
BasicProperties::default(),
)
.await
.map_err(|e| {
error!("Publish Error: {}", e);
RabbitMQError::PublishError(format!("Publish Error: {}", e))
})?
.await
.map_err(|e| {
error!("Publish Confirmation Error: {}", e);
RabbitMQError::PublishError(format!("Publish Confirmation Error: {}", e))
})?;
info!("Published message to exchange '{}' with routing key '{}'", self.exchange_name, self.routing_key);
Ok(confirmation)
}
}

74
src/server.rs
View File

@@ -1,35 +1,76 @@
use axum::{
http::StatusCode, response::{IntoResponse, Response}, routing::{get, post}, Extension, Json, Router
extract::Multipart, http::StatusCode, response::{IntoResponse, Response}, routing::{get, post}, Extension, Json, Router
};
use serde::Deserialize;
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
use zettle_db::rabbitmq::{consumer::RabbitMQConsumer, producer::RabbitMQProducer, RabbitMQConfig};
use uuid::Uuid;
use zettle_db::{models::ingress::{FileInfo, IngressContent, IngressMultipart }, rabbitmq::{consumer::RabbitMQConsumer, RabbitMQConfig}};
use zettle_db::rabbitmq::publisher::RabbitMQProducer;
use std::sync::Arc;
#[derive(Deserialize)]
struct IngressPayload {
payload: String,
}
use axum_typed_multipart::TypedMultipart;
use axum::debug_handler;
use tracing::{info, error};
async fn ingress_handler(
pub async fn ingress_handler(
Extension(producer): Extension<Arc<RabbitMQProducer>>,
Json(payload): Json<IngressPayload>
) -> Response {
info!("Received payload: {:?}", payload.payload);
match producer.publish(&payload.payload.into_bytes().to_vec()).await {
TypedMultipart(multipart_data): TypedMultipart<IngressMultipart>, // Parse form data
) -> impl IntoResponse {
info!("Received multipart data: {:?}", &multipart_data);
let file_info = if let Some(file) = multipart_data.file {
// File name or default to "data.bin" if none is provided
let file_name = file.metadata.file_name.unwrap_or(String::from("data.bin"));
let mime_type = mime_guess::from_path(&file_name)
.first_or_octet_stream()
.to_string();
let uuid = Uuid::new_v4();
let path = std::path::Path::new("/tmp").join(uuid.to_string()).join(&file_name);
// Persist the file
match file.contents.persist(&path) {
Ok(_) => {
info!("File saved at: {:?}", path);
// Generate FileInfo
let file_info = FileInfo {
uuid,
sha256: "sha-12412".to_string(),
path: path.to_string_lossy().to_string(),
mime_type,
};
Some(file_info)
}
Err(e) => {
error!("Failed to save file: {:?}", e);
return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to store file").into_response();
}
}
} else {
None // No file was uploaded
};
// Convert `IngressMultipart` to `IngressContent`
let content = match IngressContent::new(multipart_data.content, multipart_data.instructions,multipart_data.category, file_info).await {
Ok(content) => content,
Err(e) => {
error!("Error creating IngressContent: {:?}", e);
return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to create content").into_response();
}
};
// Publish content to RabbitMQ (or other system)
match producer.publish(&content).await {
Ok(_) => {
info!("Message published successfully");
"thank you".to_string().into_response()
},
"Successfully processed".to_string().into_response()
}
Err(e) => {
error!("Failed to publish message: {:?}", e);
(axum::http::StatusCode::INTERNAL_SERVER_ERROR, "Failed to publish message").into_response()
(StatusCode::INTERNAL_SERVER_ERROR, "Failed to publish message").into_response()
}
}
}
async fn queue_length_handler() -> Response {
info!("Getting queue length");
@@ -61,6 +102,7 @@ async fn queue_length_handler() -> Response {
}
}
#[tokio::main(flavor = "multi_thread", worker_threads = 2)]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Set up tracing

192
src/utils/mime.rs Normal file
View File

@@ -0,0 +1,192 @@
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tracing::info;
use url::Url;
use uuid::Uuid;
use std::path::Path;
use tokio::fs;
/// Struct to reference stored files.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Reference {
pub uuid: Uuid,
pub path: String,
}
impl Reference {
/// Creates a new Reference with a generated UUID.
pub fn new(path: String) -> Self {
Self {
uuid: Uuid::new_v4(),
path,
}
}
}
/// Enum representing different types of content.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub enum Content {
Text(String),
Url(String),
Document(Reference),
Video(Reference),
Audio(Reference),
// Extend with more variants as needed
}
impl Content {
/// Retrieves the path from a reference if the content is a Reference variant.
pub fn get_path(&self) -> Option<&str> {
match self {
Content::Document(ref r) | Content::Video(ref r) | Content::Audio(ref r) => Some(&r.path),
_ => None,
}
}
}
#[derive(Error, Debug)]
pub enum IngressContentError {
#[error("IO error occurred: {0}")]
Io(#[from] std::io::Error),
#[error("UTF-8 conversion error: {0}")]
Utf8(#[from] std::string::FromUtf8Error),
#[error("MIME type detection failed for input: {0}")]
MimeDetection(String),
#[error("Unsupported MIME type: {0}")]
UnsupportedMime(String),
#[error("URL parse error: {0}")]
UrlParse(#[from] url::ParseError),
// Add more error variants as needed.
}
#[derive(Serialize, Deserialize, Debug)]
pub struct IngressContent {
pub content: Content,
pub category: String,
pub instructions: String,
}
impl IngressContent {
/// Creates a new IngressContent instance from the given input.
///
/// # Arguments
///
/// * `input` - A string slice that holds the input content, which can be text, a file path, or a URL.
/// * `category` - A string slice representing the category of the content.
/// * `instructions` - A string slice containing instructions for processing the content.
///
/// # Returns
///
/// * `Result<IngressContent, IngressContentError>` - The result containing either the IngressContent instance or an error.
pub async fn new(
input: &str,
category: &str,
instructions: &str,
) -> Result<IngressContent, IngressContentError> {
// Check if the input is a valid URL
if let Ok(url) = Url::parse(input) {
info!("Detected URL: {}", url);
return Ok(IngressContent {
content: Content::Url(url.to_string()),
category: category.to_string(),
instructions: instructions.to_string(),
});
}
// Attempt to treat the input as a file path
if let Ok(metadata) = tokio::fs::metadata(input).await {
if metadata.is_file() {
info!("Processing as file path: {}", input);
let mime = mime_guess::from_path(input).first_or(mime::TEXT_PLAIN);
let reference = Self::store_file(input, &mime).await?;
let content = match mime.type_() {
mime::TEXT | mime::APPLICATION => Content::Document(reference),
mime::VIDEO => Content::Video(reference),
mime::AUDIO => Content::Audio(reference),
other => {
info!("Detected unsupported MIME type: {}", other);
return Err(IngressContentError::UnsupportedMime(mime.to_string()));
}
};
return Ok(IngressContent {
content,
category: category.to_string(),
instructions: instructions.to_string(),
});
}
}
// Treat the input as plain text if it's neither a URL nor a file path
info!("Treating input as plain text");
Ok(IngressContent {
content: Content::Text(input.to_string()),
category: category.to_string(),
instructions: instructions.to_string(),
})
}
/// Stores the file into 'data/' directory and returns a Reference.
async fn store_file(input_path: &str, mime: &mime::Mime) -> Result<Reference, IngressContentError> {
return Ok(Reference::new(input_path.to_string()));
// Define the data directory
let data_dir = Path::new("data/");
// Ensure 'data/' directory exists; create it if it doesn't
fs::create_dir_all(data_dir).await.map_err(IngressContentError::Io)?;
// Generate a UUID for the file
let uuid = Uuid::new_v4();
// Determine the file extension based on MIME type
// let extension = Some(mime_guess::get_mime_extensions(mime)).unwrap_or("bin");
// Create a unique filename using UUID and extension
let file_name = format!("{}.{}", uuid, extension);
// Define the full file path
let file_path = data_dir.join(&file_name);
// Copy the original file to the 'data/' directory with the new filename
fs::copy(input_path, &file_path).await.map_err(IngressContentError::Io)?;
// Return a new Reference
Ok(Reference::new(file_path.to_string_lossy().to_string()))
}
/// Example method to handle content. Implement your actual logic here.
pub fn handle_content(&self) {
match &self.content {
Content::Text(text) => {
// Handle text content
println!("Text: {}", text);
}
Content::Url(url) => {
// Handle URL content
println!("URL: {}", url);
}
Content::Document(ref reference) => {
// Handle Document content via reference
println!("Document Reference: UUID: {}, Path: {}", reference.uuid, reference.path);
// Optionally, read the file from reference.path
}
Content::Video(ref reference) => {
// Handle Video content via reference
println!("Video Reference: UUID: {}, Path: {}", reference.uuid, reference.path);
// Optionally, read the file from reference.path
}
Content::Audio(ref reference) => {
// Handle Audio content via reference
println!("Audio Reference: UUID: {}, Path: {}", reference.uuid, reference.path);
// Optionally, read the file from reference.path
}
// Handle additional content types
}
}
}

1
src/utils/mod.rs Normal file
View File

@@ -0,0 +1 @@
// pub mod mime;