feat: release build bundles templates in bin

This commit is contained in:
Per Stark
2025-03-27 08:32:21 +01:00
parent 0bc147cfc5
commit c8a97d9b52
70 changed files with 204 additions and 296 deletions

7
Cargo.lock generated
View File

@@ -2018,6 +2018,7 @@ dependencies = [
"minijinja",
"minijinja-autoreload",
"minijinja-contrib",
"minijinja-embed",
"plotly",
"serde",
"serde_json",
@@ -2841,6 +2842,12 @@ dependencies = [
"time-tz",
]
[[package]]
name = "minijinja-embed"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "290cc928e7ceb953e2e8ad2e5c6b25f5c3cf96f04697c517a2dd78e01c44f5cc"
[[package]]
name = "minimal-lexical"
version = "0.2.1"

View File

@@ -31,3 +31,7 @@ tempfile = "3.12.0"
url = { version = "2.5.2", features = ["serde"] }
uuid = { version = "1.10.0", features = ["v4", "serde"] }
minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
minijinja-autoreload = "2.5.0"
minijinja-embed = { version = "2.8.0" }
minijinja-contrib = { version = "2.6.0", features = ["datetime", "timezone"] }

View File

@@ -1 +0,0 @@

View File

@@ -1,3 +1,3 @@
pub mod config;
pub mod embedding;
pub mod mailer;
pub mod template_engine;

View File

@@ -0,0 +1,92 @@
pub use minijinja::{path_loader, Environment, Value};
pub use minijinja_autoreload::AutoReloader;
pub use minijinja_contrib;
pub use minijinja_embed;
use std::sync::Arc;
#[derive(Clone)]
pub enum TemplateEngine {
// Use AutoReload for debug builds (debug_assertions is true)
#[cfg(debug_assertions)]
AutoReload(Arc<AutoReloader>),
// Use Embedded for release builds (debug_assertions is false)
#[cfg(not(debug_assertions))]
Embedded(Arc<Environment<'static>>),
}
#[macro_export]
macro_rules! create_template_engine {
// Macro takes the relative path to the templates dir as input
($relative_path:expr) => {{
// Code for debug builds (AutoReload)
#[cfg(debug_assertions)]
{
// These lines execute in the CALLING crate's context
let crate_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let template_path = crate_dir.join($relative_path);
let reloader = $crate::utils::template_engine::AutoReloader::new(move |notifier| {
let mut env = $crate::utils::template_engine::Environment::new();
env.set_loader($crate::utils::template_engine::path_loader(&template_path));
notifier.set_fast_reload(true);
notifier.watch_path(&template_path, true);
// Add contrib filters/functions
$crate::utils::template_engine::minijinja_contrib::add_to_environment(&mut env);
Ok(env)
});
$crate::utils::template_engine::TemplateEngine::AutoReload(std::sync::Arc::new(
reloader,
))
}
// Code for release builds (Embedded)
#[cfg(not(debug_assertions))]
{
// These lines also execute in the CALLING crate's context
let mut env = $crate::utils::template_engine::Environment::new();
$crate::utils::template_engine::minijinja_embed::load_templates!(&mut env);
// Add contrib filters/functions
$crate::utils::template_engine::minijinja_contrib::add_to_environment(&mut env);
$crate::utils::template_engine::TemplateEngine::Embedded(std::sync::Arc::new(env))
}
}};
}
impl TemplateEngine {
pub fn render(&self, name: &str, ctx: &Value) -> Result<String, minijinja::Error> {
match self {
// Only compile this arm for debug builds
#[cfg(debug_assertions)]
TemplateEngine::AutoReload(reloader) => {
let env = reloader.acquire_env()?;
env.get_template(name)?.render(ctx)
}
// Only compile this arm for release builds
#[cfg(not(debug_assertions))]
TemplateEngine::Embedded(env) => env.get_template(name)?.render(ctx),
}
}
pub fn render_block(
&self,
template_name: &str,
block_name: &str,
context: &Value,
) -> Result<String, minijinja::Error> {
match self {
// Only compile this arm for debug builds
#[cfg(debug_assertions)]
TemplateEngine::AutoReload(reloader) => {
let env = reloader.acquire_env()?;
let template = env.get_template(template_name)?;
let mut state = template.eval_to_state(context)?;
state.render_block(block_name)
}
// Only compile this arm for release builds
#[cfg(not(debug_assertions))]
TemplateEngine::Embedded(env) => {
let template = env.get_template(template_name)?;
let mut state = template.eval_to_state(context)?;
state.render_block(block_name)
}
}
}
}

View File

@@ -24,6 +24,7 @@ async-stream = "0.3.6"
json-stream-parser = "0.1.4"
minijinja = { version = "2.5.0", features = ["loader", "multi_template"] }
minijinja-autoreload = "2.5.0"
minijinja-embed = { version = "2.8.0" }
minijinja-contrib = { version = "2.6.0", features = ["datetime", "timezone"] }
plotly = "0.12.1"
surrealdb = "2.0.4"
@@ -32,3 +33,6 @@ chrono-tz = "0.10.1"
common = { path = "../common" }
composite-retrieval = { path = "../composite-retrieval" }
[build-dependencies]
minijinja-embed = { version = "2.8.0" }

View File

@@ -0,0 +1,12 @@
fn main() {
// Get the build profile ("debug" or "release")
let profile = std::env::var("PROFILE").unwrap_or_else(|_| "debug".to_string());
// Embed templates only for release builds
if profile == "release" {
// Embed templates from the "templates" directory relative to CARGO_MANIFEST_DIR
minijinja_embed::embed_templates!("templates");
} else {
println!("cargo:info=Build: Skipping template embedding for debug build.");
}
}

View File

@@ -1,10 +1,9 @@
use axum_session::SessionStore;
use axum_session_surreal::SessionSurrealPool;
use common::create_template_engine;
use common::storage::db::SurrealDbClient;
use common::utils::config::AppConfig;
use minijinja::{path_loader, Environment};
use minijinja_autoreload::AutoReloader;
use std::path::PathBuf;
use common::utils::template_engine::TemplateEngine;
use std::sync::Arc;
use surrealdb::engine::any::Any;
@@ -12,22 +11,13 @@ use surrealdb::engine::any::Any;
pub struct HtmlState {
pub db: Arc<SurrealDbClient>,
pub openai_client: Arc<async_openai::Client<async_openai::config::OpenAIConfig>>,
pub templates: Arc<AutoReloader>,
pub templates: Arc<TemplateEngine>,
pub session_store: Arc<SessionStore<SessionSurrealPool<Any>>>,
}
impl HtmlState {
pub async fn new(config: &AppConfig) -> Result<Self, Box<dyn std::error::Error>> {
let reloader = AutoReloader::new(move |notifier| {
let template_path = get_templates_dir();
let mut env = Environment::new();
env.set_loader(path_loader(&template_path));
notifier.set_fast_reload(true);
notifier.watch_path(&template_path, true);
minijinja_contrib::add_to_environment(&mut env);
Ok(env)
});
let template_engine = create_template_engine!("templates");
let surreal_db_client = Arc::new(
SurrealDbClient::new(
@@ -48,7 +38,7 @@ impl HtmlState {
let app_state = HtmlState {
db: surreal_db_client.clone(),
templates: Arc::new(reloader),
templates: Arc::new(template_engine),
openai_client: openai_client.clone(),
session_store,
};
@@ -56,23 +46,3 @@ impl HtmlState {
Ok(app_state)
}
}
pub fn get_workspace_root() -> PathBuf {
// Starts from CARGO_MANIFEST_DIR (e.g., /project/crates/html-router/)
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
// Navigate up to /path/to/project/crates
let crates_dir = manifest_dir
.parent()
.expect("Failed to find parent of manifest directory");
// Navigate up to workspace root
crates_dir
.parent()
.expect("Failed to find workspace root")
.to_path_buf()
}
pub fn get_templates_dir() -> PathBuf {
get_workspace_root().join("templates")
}

View File

@@ -1,3 +1,4 @@
use crate::html_state::HtmlState;
use axum::{
extract::State,
http::StatusCode,
@@ -6,19 +7,15 @@ use axum::{
};
use common::error::AppError;
use minijinja::{context, Value};
use minijinja_autoreload::AutoReloader;
use serde::Serialize;
use std::sync::Arc;
use tracing::error;
use crate::html_state::HtmlState;
// Enum for template types
#[derive(Clone)]
pub enum TemplateKind {
Full(String), // Full page template
Partial(String, String), // Template name, block name
Error(StatusCode), // Error template with status code
Redirect(String), // Redirect
Full(String),
Partial(String, String),
Error(StatusCode),
Redirect(String),
}
#[derive(Clone)]
@@ -53,14 +50,12 @@ impl TemplateResponse {
error => error,
description => description
};
Self {
template_kind: TemplateKind::Error(status),
context: ctx,
}
}
// Convenience methods for common errors
pub fn not_found() -> Self {
Self::error(
StatusCode::NOT_FOUND,
@@ -118,25 +113,33 @@ struct TemplateStateWrapper {
impl IntoResponse for TemplateStateWrapper {
fn into_response(self) -> Response {
let templates = self.state.templates;
let template_engine = &self.state.templates;
match &self.template_response.template_kind {
TemplateKind::Full(name) => {
render_template(name, self.template_response.context, templates)
match template_engine.render(name, &self.template_response.context) {
Ok(html) => Html(html).into_response(),
Err(e) => {
error!("Failed to render template: {:?}", e);
(StatusCode::INTERNAL_SERVER_ERROR, fallback_error()).into_response()
}
}
}
TemplateKind::Partial(name, block) => {
render_block(name, block, self.template_response.context, templates)
TemplateKind::Partial(template, block) => {
match template_engine.render_block(template, block, &self.template_response.context)
{
Ok(html) => Html(html).into_response(),
Err(e) => {
error!("Failed to render block: {:?}", e);
(StatusCode::INTERNAL_SERVER_ERROR, fallback_error()).into_response()
}
}
}
TemplateKind::Error(status) => {
let html = match try_render_template(
"errors/error.html",
self.template_response.context,
templates,
) {
Ok(html_string) => Html(html_string),
Err(_) => fallback_error(),
};
(*status, html).into_response()
match template_engine.render("errors/error.html", &self.template_response.context) {
Ok(html) => (*status, Html(html)).into_response(),
Err(_) => (*status, fallback_error()).into_response(),
}
}
TemplateKind::Redirect(path) => {
(StatusCode::OK, [(axum_htmx::HX_REDIRECT, path.clone())], "").into_response()
@@ -145,93 +148,11 @@ impl IntoResponse for TemplateStateWrapper {
}
}
// Helper functions for rendering with error handling
fn render_template(name: &str, context: Value, templates: Arc<AutoReloader>) -> Response {
match try_render_template(name, context, templates.clone()) {
Ok(html) => Html(html).into_response(),
Err(_) => fallback_error().into_response(),
}
}
fn render_block(name: &str, block: &str, context: Value, templates: Arc<AutoReloader>) -> Response {
match try_render_block(name, block, context, templates.clone()) {
Ok(html) => Html(html).into_response(),
Err(_) => fallback_error().into_response(),
}
}
fn try_render_template(
template_name: &str,
context: Value,
templates: Arc<AutoReloader>,
) -> Result<String, ()> {
let env = templates.acquire_env().map_err(|e| {
tracing::error!("Environment error: {:?}", e);
()
})?;
let tmpl = env.get_template(template_name).map_err(|e| {
tracing::error!("Template error: {:?}", e);
()
})?;
tmpl.render(context).map_err(|e| {
tracing::error!("Render error: {:?}", e);
()
})
}
fn try_render_block(
template_name: &str,
block: &str,
context: Value,
templates: Arc<AutoReloader>,
) -> Result<String, ()> {
let env = templates.acquire_env().map_err(|e| {
tracing::error!("Environment error: {:?}", e);
()
})?;
let tmpl = env.get_template(template_name).map_err(|e| {
tracing::error!("Template error: {:?}", e);
()
})?;
let mut state = tmpl.eval_to_state(context).map_err(|e| {
tracing::error!("Eval error: {:?}", e);
()
})?;
state.render_block(block).map_err(|e| {
tracing::error!("Block render error: {:?}", e);
()
})
}
fn fallback_error() -> Html<String> {
Html(
r#"
<html>
<body>
<div class="container mx-auto p-4">
<h1 class="text-4xl text-error">Error</h1>
<p class="mt-4">Sorry, something went wrong displaying this page.</p>
</div>
</body>
</html>
"#
.to_string(),
)
}
pub async fn with_template_response(
State(state): State<HtmlState>,
response: Response,
) -> Response {
// Clone the TemplateResponse from extensions
let template_response = response.extensions().get::<TemplateResponse>().cloned();
if let Some(template_response) = template_response {
if let Some(template_response) = response.extensions().get::<TemplateResponse>().cloned() {
TemplateStateWrapper {
state,
template_response,
@@ -242,40 +163,60 @@ pub async fn with_template_response(
}
}
// Define HtmlError
#[derive(Debug)]
pub enum HtmlError {
AppError(AppError),
TemplateError(String),
}
// Conversion from AppError to HtmlError
impl From<AppError> for HtmlError {
fn from(err: AppError) -> Self {
HtmlError::AppError(err)
}
}
// Conversion for database error to HtmlError
impl From<surrealdb::Error> for HtmlError {
fn from(err: surrealdb::Error) -> Self {
HtmlError::AppError(AppError::from(err))
}
}
impl From<minijinja::Error> for HtmlError {
fn from(err: minijinja::Error) -> Self {
HtmlError::TemplateError(err.to_string())
}
}
impl IntoResponse for HtmlError {
fn into_response(self) -> Response {
match self {
HtmlError::AppError(err) => {
let template_response = match err {
AppError::NotFound(_) => TemplateResponse::not_found(),
AppError::Auth(_) => TemplateResponse::unauthorized(),
AppError::Validation(msg) => TemplateResponse::bad_request(&msg),
_ => {
tracing::error!("Internal error: {:?}", err);
TemplateResponse::server_error()
}
};
template_response.into_response()
HtmlError::AppError(err) => match err {
AppError::NotFound(_) => TemplateResponse::not_found().into_response(),
AppError::Auth(_) => TemplateResponse::unauthorized().into_response(),
AppError::Validation(msg) => TemplateResponse::bad_request(&msg).into_response(),
_ => {
error!("Internal error: {:?}", err);
TemplateResponse::server_error().into_response()
}
},
HtmlError::TemplateError(err) => {
error!("Template error: {}", err);
TemplateResponse::server_error().into_response()
}
}
}
}
fn fallback_error() -> String {
r#"
<html>
<body>
<div class="container mx-auto p-4">
<h1 class="text-4xl text-error">Error</h1>
<p class="mt-4">Sorry, something went wrong displaying this page.</p>
</div>
</body>
</html>
"#
.to_string()
}

View File

@@ -22,23 +22,24 @@ use futures::{
Stream, StreamExt, TryStreamExt,
};
use json_stream_parser::JsonStreamParser;
use minijinja::Value;
use serde::{Deserialize, Serialize};
use serde_json::from_str;
use surrealdb::{engine::any::Any, Surreal};
use tokio::sync::{mpsc::channel, Mutex};
use tracing::{error, debug};
use tracing::{debug, error};
use common::storage::{
db::SurrealDbClient,
types::{
conversation::Conversation,
message::{Message, MessageRole},
user::User,
system_settings::SystemSettings,
user::User,
},
};
use crate::{html_state::HtmlState, routes::render_template};
use crate::html_state::HtmlState;
// Error handling function
fn create_error_stream(
@@ -272,17 +273,13 @@ pub async fn get_response_stream(
}
// Render template with references
match render_template(
match state.templates.render(
"chat/reference_list.html",
ReferenceData { message },
state.templates.clone(),
&Value::from_serialize(ReferenceData { message }),
) {
Ok(html) => {
// Extract the String from Html<String>
let html_string = html.0;
// Return the rendered HTML
Ok(Event::default().event("references").data(html_string))
Ok(Event::default().event("references").data(html))
}
Err(_) => {
// Handle template rendering error

View File

@@ -1,10 +1,3 @@
use std::sync::Arc;
use axum::response::Html;
use minijinja_autoreload::AutoReloader;
use crate::middlewares::response_middleware::HtmlError;
pub mod account;
pub mod admin;
pub mod auth;
@@ -14,20 +7,3 @@ pub mod index;
pub mod ingestion;
pub mod knowledge;
pub mod search;
// Helper function for render_template
pub fn render_template<T>(
template_name: &str,
context: T,
templates: Arc<AutoReloader>,
) -> Result<Html<String>, HtmlError>
where
T: serde::Serialize,
{
let env = templates.acquire_env().unwrap();
let tmpl = env.get_template(template_name).unwrap();
let context = minijinja::Value::from_serialize(&context);
let output = tmpl.render(context).unwrap();
Ok(Html(output))
}

View File

Before

Width:  |  Height:  |  Size: 556 B

After

Width:  |  Height:  |  Size: 556 B

View File

Before

Width:  |  Height:  |  Size: 326 B

After

Width:  |  Height:  |  Size: 326 B

View File

Before

Width:  |  Height:  |  Size: 572 B

After

Width:  |  Height:  |  Size: 572 B

View File

Before

Width:  |  Height:  |  Size: 617 B

After

Width:  |  Height:  |  Size: 617 B

View File

Before

Width:  |  Height:  |  Size: 486 B

After

Width:  |  Height:  |  Size: 486 B

View File

Before

Width:  |  Height:  |  Size: 460 B

After

Width:  |  Height:  |  Size: 460 B

View File

Before

Width:  |  Height:  |  Size: 679 B

After

Width:  |  Height:  |  Size: 679 B

View File

Before

Width:  |  Height:  |  Size: 244 B

After

Width:  |  Height:  |  Size: 244 B

View File

Before

Width:  |  Height:  |  Size: 371 B

After

Width:  |  Height:  |  Size: 371 B

View File

Before

Width:  |  Height:  |  Size: 301 B

After

Width:  |  Height:  |  Size: 301 B

View File

@@ -3,13 +3,15 @@
<input type="checkbox" class="theme-controller" value="dark" />
<!-- sun icon -->
<svg class="swap-off h-5 w-5 fill-current" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
<svg width="20" height="20" class="swap-off h-5 w-5 fill-current" xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24">
<path
d="M5.64,17l-.71.71a1,1,0,0,0,0,1.41,1,1,0,0,0,1.41,0l.71-.71A1,1,0,0,0,5.64,17ZM5,12a1,1,0,0,0-1-1H3a1,1,0,0,0,0,2H4A1,1,0,0,0,5,12Zm7-7a1,1,0,0,0,1-1V3a1,1,0,0,0-2,0V4A1,1,0,0,0,12,5ZM5.64,7.05a1,1,0,0,0,.7.29,1,1,0,0,0,.71-.29,1,1,0,0,0,0-1.41l-.71-.71A1,1,0,0,0,4.93,6.34Zm12,.29a1,1,0,0,0,.7-.29l.71-.71a1,1,0,1,0-1.41-1.41L17,5.64a1,1,0,0,0,0,1.41A1,1,0,0,0,17.66,7.34ZM21,11H20a1,1,0,0,0,0,2h1a1,1,0,0,0,0-2Zm-9,8a1,1,0,0,0-1,1v1a1,1,0,0,0,2,0V20A1,1,0,0,0,12,19ZM18.36,17A1,1,0,0,0,17,18.36l.71.71a1,1,0,0,0,1.41,0,1,1,0,0,0,0-1.41ZM12,6.5A5.5,5.5,0,1,0,17.5,12,5.51,5.51,0,0,0,12,6.5Zm0,9A3.5,3.5,0,1,1,15.5,12,3.5,3.5,0,0,1,12,15.5Z" />
</svg>
<!-- moon icon -->
<svg class="swap-on h-5 w-5 fill-current" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
<svg width="20" height="20" class="swap-on h-5 w-5 fill-current" xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24">
<path
d="M21.64,13a1,1,0,0,0-1.05-.14,8.05,8.05,0,0,1-3.37.73A8.15,8.15,0,0,1,9.08,5.49a8.59,8.59,0,0,1,.25-2A1,1,0,0,0,8,2.36,10.14,10.14,0,1,0,22,14.05,1,1,0,0,0,21.64,13Zm-9.5,6.69A8.14,8.14,0,0,1,7.08,5.22v.27A10.15,10.15,0,0,0,17.22,15.63a9.79,9.79,0,0,0,2.1-.22A8.11,8.11,0,0,1,12.14,19.73Z" />
</svg>

View File

@@ -1,77 +0,0 @@
{# email_verification.html #}
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Email Verification</title>
<style>
body {
font-family: Arial, sans-serif;
line-height: 1.6;
color: #333333;
margin: 0;
padding: 0;
}
.container {
max-width: 600px;
margin: 0 auto;
padding: 20px;
text-align: center;
}
.card {
background-color: #ffffff;
border-radius: 8px;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
padding: 30px;
margin: 20px auto;
max-width: 400px;
}
.verification-code {
font-size: 32px;
font-weight: bold;
color: #2c3e50;
letter-spacing: 2px;
margin: 20px 0;
padding: 10px;
background-color: #f8f9fa;
border-radius: 4px;
}
.footer {
margin-top: 30px;
font-size: 12px;
color: #666666;
}
.header {
margin-bottom: 30px;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>Email Verification</h1>
</div>
<div class="card">
<p>Hello {{ name }},</p>
<p>Please use the following verification code to complete your registration:</p>
<div class="verification-code">
{{ verification_code }}
</div>
<p>This code will expire in 30 minutes.</p>
</div>
<div class="footer">
<p><strong>Security Notice:</strong> If you didn't request this verification code, please ignore this email. Someone might have entered your email address by mistake.</p>
<p>For your security, never share this code with anyone, including those claiming to be from our support team.</p>
<p>This is an automated message, please do not reply to this email.</p>
<hr style="margin: 20px 0; border: none; border-top: 1px solid #eee;">
<p>© 2024 Your Company Name. All rights reserved.</p>
</div>
</div>
</body>
</html>

View File

@@ -1,20 +0,0 @@
{# email_verification.txt #}
Hello {{ name }},
Thank you for registering. To complete your registration, please use the following verification code:
{{ verification_code }}
This code will expire in 30 minutes.
IMPORTANT SECURITY INFORMATION:
- If you didn't request this verification code, please ignore this email.
- Never share this code with anyone, including those claiming to be from our support team.
- This is an automated message, please do not reply to this email.
For your security, if you did not initiate this request, you can safely ignore this email. Someone might have entered your email address by mistake.
Best regards,
Your Company Name
© 2024 Your Company Name. All rights reserved.

11
todo.md
View File

@@ -1,13 +1,12 @@
\[\] archive ingressed webpage
\[\] openai api key in config
\[x\] option to set models, query and processing
\[x\] template customization?
\[\] configs primarily get envs
\[\] filtering on categories
\[\] integrate assets folder in release build
\[\] integrate templates in release build
\[\] markdown rendering in client
\[\] openai api key in config
\[\] three js graph explorer
\[\] three js vector explorer
\[\] integrate templates in release build
\[\] integrate assets folder in release build
\[x\] add user_id to ingress objects
\[x\] admin controls re registration
\[x\] chat functionality
@@ -22,9 +21,11 @@
\[x\] link to ingressed urls or archives
\[x\] macro for pagedata?
\[x\] on updates of knowledgeentity create new embeddings
\[x\] option to set models, query and processing
\[x\] redirects
\[x\] restrict retrieval to users own objects
\[x\] smoothie_dom test
\[x\] template customization?
\[x\] templating
\[x\] user id to fileinfo and data path?
\[x\] view content