Files
yaak-mountain-loop/crates/yaak-models/src/blob_manager.rs
T

495 lines
16 KiB
Rust

use crate::error::Error::GenericError;
use crate::error::Result;
use crate::util::generate_prefixed_id;
use include_dir::{Dir, include_dir};
use log::{debug, info};
use rusqlite::{OptionalExtension, Transaction, TransactionBehavior, params};
use std::ops::Deref;
use yaak_database::{ConnectionOrTx, SqlitePool};
static BLOB_MIGRATIONS_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/blob_migrations");
/// A chunk of body data stored in the blob database.
#[derive(Debug, Clone)]
pub struct BodyChunk {
pub id: String,
pub body_id: String,
pub chunk_index: i32,
pub data: Vec<u8>,
}
impl BodyChunk {
pub fn new(body_id: impl Into<String>, chunk_index: i32, data: Vec<u8>) -> Self {
Self { id: generate_prefixed_id("bc"), body_id: body_id.into(), chunk_index, data }
}
}
/// Manages the blob database: a reader pool and a single writer, for the
/// same reason as [`crate::query_manager::QueryManager`].
// Pools are internally synchronized — don't wrap them in a Mutex. A Mutex held across the
// blocking `get()` serializes every blob access behind the slowest waiter, freezing the
// whole app whenever the pool is exhausted.
#[derive(Debug, Clone)]
pub struct BlobManager {
readers: SqlitePool,
writer: SqlitePool,
}
impl BlobManager {
/// `writer` must be a pool with a single connection.
pub fn new(readers: SqlitePool, writer: SqlitePool) -> Self {
Self { readers, writer }
}
/// A read handle from the reader pool.
pub fn connect(&self) -> BlobContext<'_> {
let conn = self.readers.get().expect("Failed to get blob DB connection from pool");
BlobContext { conn: ConnectionOrTx::Connection(conn) }
}
/// Run `func` in a transaction on the writer connection.
pub fn with_tx<T, E>(
&self,
func: impl FnOnce(&BlobWriter) -> std::result::Result<T, E>,
) -> std::result::Result<T, E>
where
E: From<crate::error::Error>,
{
let conn = self.writer.get().map_err(crate::error::Error::SqlPoolError)?;
let tx = Transaction::new_unchecked(&conn, TransactionBehavior::Immediate)
.map_err(crate::error::Error::SqlError)?;
let writer = BlobWriter { ctx: BlobContext { conn: ConnectionOrTx::Transaction(&tx) } };
match func(&writer) {
Ok(val) => {
tx.commit().map_err(|e| {
GenericError(format!("Failed to commit blob transaction {e:?}"))
})?;
Ok(val)
}
Err(e) => {
tx.rollback().map_err(|e| {
GenericError(format!("Failed to rollback blob transaction {e:?}"))
})?;
Err(e)
}
}
}
}
/// Read handle for the blob database.
pub struct BlobContext<'a> {
conn: ConnectionOrTx<'a>,
}
impl<'a> BlobContext<'a> {
/// Get all chunks for a body, ordered by chunk_index.
pub fn get_chunks(&self, body_id: &str) -> Result<Vec<BodyChunk>> {
let mut stmt = self.conn.prepare(
"SELECT id, body_id, chunk_index, data FROM body_chunks
WHERE body_id = ?1 ORDER BY chunk_index ASC",
)?;
let chunks = stmt
.query_map(params![body_id], |row| {
Ok(BodyChunk {
id: row.get(0)?,
body_id: row.get(1)?,
chunk_index: row.get(2)?,
data: row.get(3)?,
})
})?
.collect::<std::result::Result<Vec<_>, _>>()?;
Ok(chunks)
}
/// List all distinct body IDs in the blob database.
pub fn list_body_ids(&self) -> Result<Vec<String>> {
let mut stmt = self.conn.prepare("SELECT DISTINCT body_id FROM body_chunks")?;
let ids = stmt
.query_map([], |row| row.get(0))?
.collect::<std::result::Result<Vec<String>, _>>()?;
Ok(ids)
}
/// Get total size of a body without loading data.
pub fn get_body_size(&self, body_id: &str) -> Result<usize> {
let size: i64 = self
.conn
.resolve()
.query_row(
"SELECT COALESCE(SUM(LENGTH(data)), 0) FROM body_chunks WHERE body_id = ?1",
params![body_id],
|row| row.get(0),
)
.unwrap_or(0);
Ok(size as usize)
}
/// Check if a body exists.
pub fn body_exists(&self, body_id: &str) -> Result<bool> {
let count: i64 = self
.conn
.resolve()
.query_row(
"SELECT COUNT(*) FROM body_chunks WHERE body_id = ?1",
params![body_id],
|row| row.get(0),
)
.unwrap_or(0);
Ok(count > 0)
}
}
/// Write handle for the blob database. Derefs to [`BlobContext`] for reads.
pub struct BlobWriter<'a> {
ctx: BlobContext<'a>,
}
impl<'a> Deref for BlobWriter<'a> {
type Target = BlobContext<'a>;
fn deref(&self) -> &BlobContext<'a> {
&self.ctx
}
}
impl<'a> BlobWriter<'a> {
/// Insert a single chunk.
pub fn insert_chunk(&self, chunk: &BodyChunk) -> Result<()> {
self.conn.execute(
"INSERT INTO body_chunks (id, body_id, chunk_index, data) VALUES (?1, ?2, ?3, ?4)",
params![chunk.id, chunk.body_id, chunk.chunk_index, chunk.data],
)?;
Ok(())
}
/// Delete all chunks for a body.
pub fn delete_chunks(&self, body_id: &str) -> Result<()> {
self.conn.execute("DELETE FROM body_chunks WHERE body_id = ?1", params![body_id])?;
Ok(())
}
/// Delete all chunks matching a body_id prefix (e.g., "rs_abc123.%" to delete all
/// bodies for a response).
pub fn delete_chunks_like(&self, body_id_prefix: &str) -> Result<()> {
self.conn
.execute("DELETE FROM body_chunks WHERE body_id LIKE ?1", params![body_id_prefix])?;
Ok(())
}
}
/// Size of each stored chunk. Bodies arrive in small reads (8 to 16 KiB is typical), so they
/// are grouped into chunks of this size to avoid thousands of tiny inserts for large bodies.
pub const BODY_CHUNK_SIZE: usize = 1024 * 1024;
/// Writes one body as it streams, grouping its pieces into [`BODY_CHUNK_SIZE`] chunks. Each
/// full chunk is inserted as soon as it fills; [`BodyChunkWriter::finish`] inserts the rest.
pub struct BodyChunkWriter {
blob_manager: BlobManager,
body_id: String,
buf: Vec<u8>,
chunk_index: i32,
total_bytes: usize,
}
impl BodyChunkWriter {
pub fn new(blob_manager: BlobManager, body_id: impl Into<String>) -> Self {
Self {
blob_manager,
body_id: body_id.into(),
buf: Vec::new(),
chunk_index: 0,
total_bytes: 0,
}
}
pub fn write(&mut self, data: &[u8]) -> Result<()> {
self.total_bytes += data.len();
self.buf.extend_from_slice(data);
while self.buf.len() >= BODY_CHUNK_SIZE {
let data = self.buf.drain(..BODY_CHUNK_SIZE).collect();
self.insert(data)?;
}
Ok(())
}
/// Inserts the partial last chunk and returns the bytes written in all.
pub fn finish(mut self) -> Result<usize> {
if !self.buf.is_empty() {
let data = std::mem::take(&mut self.buf);
self.insert(data)?;
}
Ok(self.total_bytes)
}
fn insert(&mut self, data: Vec<u8>) -> Result<()> {
let chunk = BodyChunk::new(&self.body_id, self.chunk_index, data);
self.blob_manager.with_tx(|b| b.insert_chunk(&chunk))?;
self.chunk_index += 1;
Ok(())
}
}
/// Run migrations for the blob database.
pub fn migrate_blob_db(pool: &SqlitePool) -> Result<()> {
info!("Running blob database migrations");
// Create migrations tracking table
pool.get()?.execute(
"CREATE TABLE IF NOT EXISTS _blob_migrations (
version TEXT PRIMARY KEY,
description TEXT NOT NULL,
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
)",
[],
)?;
// Read and sort all .sql files
let mut entries: Vec<_> = BLOB_MIGRATIONS_DIR
.entries()
.iter()
.filter(|e| e.path().extension().map(|ext| ext == "sql").unwrap_or(false))
.collect();
entries.sort_by_key(|e| e.path());
let mut ran_migrations = 0;
for entry in &entries {
let filename = entry.path().file_name().unwrap().to_str().unwrap();
let version = filename.split('_').next().unwrap();
// Check if already applied
let already_applied: Option<i64> = pool
.get()?
.query_row("SELECT 1 FROM _blob_migrations WHERE version = ?", [version], |r| r.get(0))
.optional()?;
if already_applied.is_some() {
debug!("Skipping already applied blob migration: {}", filename);
continue;
}
let sql =
entry.as_file().unwrap().contents_utf8().expect("Failed to read blob migration file");
info!("Applying blob migration: {}", filename);
let conn = pool.get()?;
conn.execute_batch(sql)?;
// Record migration
conn.execute(
"INSERT INTO _blob_migrations (version, description) VALUES (?, ?)",
params![version, filename],
)?;
ran_migrations += 1;
}
if ran_migrations == 0 {
info!("No blob migrations to run");
} else {
info!("Ran {} blob migration(s)", ran_migrations);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
fn create_test_manager() -> BlobManager {
let manager = r2d2_sqlite::SqliteConnectionManager::memory();
let pool = r2d2::Pool::builder().max_size(1).build(manager).unwrap();
migrate_blob_db(&pool).unwrap();
BlobManager::new(pool.clone(), pool)
}
fn insert(manager: &BlobManager, chunks: &[BodyChunk]) {
manager
.with_tx(|b| {
for c in chunks {
b.insert_chunk(c)?;
}
Ok::<_, Error>(())
})
.unwrap();
}
#[test]
fn test_insert_and_get_chunks() {
let manager = create_test_manager();
let body_id = "rs_test123.request";
insert(
&manager,
&[
BodyChunk::new(body_id, 0, b"Hello, ".to_vec()),
BodyChunk::new(body_id, 1, b"World!".to_vec()),
],
);
let chunks = manager.connect().get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].chunk_index, 0);
assert_eq!(chunks[0].data, b"Hello, ");
assert_eq!(chunks[1].chunk_index, 1);
assert_eq!(chunks[1].data, b"World!");
}
#[test]
fn test_get_chunks_ordered_by_index() {
let manager = create_test_manager();
let body_id = "rs_test123.request";
// Insert out of order
insert(
&manager,
&[
BodyChunk::new(body_id, 2, b"C".to_vec()),
BodyChunk::new(body_id, 0, b"A".to_vec()),
BodyChunk::new(body_id, 1, b"B".to_vec()),
],
);
let chunks = manager.connect().get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].data, b"A");
assert_eq!(chunks[1].data, b"B");
assert_eq!(chunks[2].data, b"C");
}
#[test]
fn test_delete_chunks() {
let manager = create_test_manager();
let body_id = "rs_test123.request";
insert(&manager, &[BodyChunk::new(body_id, 0, b"data".to_vec())]);
assert!(manager.connect().body_exists(body_id).unwrap());
manager.with_tx(|b| b.delete_chunks(body_id)).unwrap();
let ctx = manager.connect();
assert!(!ctx.body_exists(body_id).unwrap());
assert_eq!(ctx.get_chunks(body_id).unwrap().len(), 0);
}
#[test]
fn test_delete_chunks_like() {
let manager = create_test_manager();
// Insert chunks for same response but different body types
insert(
&manager,
&[
BodyChunk::new("rs_abc.request", 0, b"req".to_vec()),
BodyChunk::new("rs_abc.response", 0, b"resp".to_vec()),
BodyChunk::new("rs_other.request", 0, b"other".to_vec()),
],
);
// Delete all bodies for rs_abc
manager.with_tx(|b| b.delete_chunks_like("rs_abc.%")).unwrap();
let ctx = manager.connect();
assert!(!ctx.body_exists("rs_abc.request").unwrap());
assert!(!ctx.body_exists("rs_abc.response").unwrap());
assert!(ctx.body_exists("rs_other.request").unwrap());
}
#[test]
fn test_get_body_size() {
let manager = create_test_manager();
let body_id = "rs_test123.request";
insert(
&manager,
&[
BodyChunk::new(body_id, 0, b"Hello".to_vec()),
BodyChunk::new(body_id, 1, b"World".to_vec()),
],
);
let size = manager.connect().get_body_size(body_id).unwrap();
assert_eq!(size, 10); // "Hello" + "World" = 10 bytes
}
#[test]
fn test_get_body_size_empty() {
let manager = create_test_manager();
let size = manager.connect().get_body_size("nonexistent").unwrap();
assert_eq!(size, 0);
}
#[test]
fn test_body_exists() {
let manager = create_test_manager();
assert!(!manager.connect().body_exists("rs_test.request").unwrap());
insert(&manager, &[BodyChunk::new("rs_test.request", 0, b"data".to_vec())]);
assert!(manager.connect().body_exists("rs_test.request").unwrap());
}
#[test]
fn test_multiple_bodies_isolated() {
let manager = create_test_manager();
insert(
&manager,
&[
BodyChunk::new("body1", 0, b"data1".to_vec()),
BodyChunk::new("body2", 0, b"data2".to_vec()),
],
);
let ctx = manager.connect();
let chunks1 = ctx.get_chunks("body1").unwrap();
let chunks2 = ctx.get_chunks("body2").unwrap();
assert_eq!(chunks1.len(), 1);
assert_eq!(chunks1[0].data, b"data1");
assert_eq!(chunks2.len(), 1);
assert_eq!(chunks2[0].data, b"data2");
}
#[test]
fn test_chunk_writer_groups_pieces() {
let manager = create_test_manager();
let body_id = "fl_test.response";
let piece = vec![7u8; 16 * 1024];
let pieces = BODY_CHUNK_SIZE * 2 / piece.len() + 3;
let mut writer = BodyChunkWriter::new(manager.clone(), body_id);
for _ in 0..pieces {
writer.write(&piece).unwrap();
}
assert_eq!(manager.connect().get_chunks(body_id).unwrap().len(), 2);
assert_eq!(writer.finish().unwrap(), pieces * piece.len());
let chunks = manager.connect().get_chunks(body_id).unwrap();
let sizes: Vec<_> = chunks.iter().map(|c| c.data.len()).collect();
assert_eq!(sizes, vec![BODY_CHUNK_SIZE, BODY_CHUNK_SIZE, 3 * piece.len()]);
assert_eq!(chunks.iter().map(|c| c.chunk_index).collect::<Vec<_>>(), vec![0, 1, 2]);
}
#[test]
fn test_chunk_writer_empty_body() {
let manager = create_test_manager();
let writer = BodyChunkWriter::new(manager.clone(), "fl_empty.request");
assert_eq!(writer.finish().unwrap(), 0);
assert!(!manager.connect().body_exists("fl_empty.request").unwrap());
}
#[test]
fn test_large_chunk() {
let manager = create_test_manager();
// 1MB chunk
let large_data: Vec<u8> = (0..1024 * 1024).map(|i| (i % 256) as u8).collect();
let body_id = "rs_large.request";
insert(&manager, &[BodyChunk::new(body_id, 0, large_data.clone())]);
let ctx = manager.connect();
let chunks = ctx.get_chunks(body_id).unwrap();
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].data, large_data);
assert_eq!(ctx.get_body_size(body_id).unwrap(), 1024 * 1024);
}
}