mirror of
https://github.com/perstarkse/minne.git
synced 2026-03-21 00:49:54 +01:00
wip datetime impl
This commit is contained in:
Per Stark
150
Cargo.lock
generated
150
Cargo.lock
generated
@@ -280,7 +280,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"static_assertions_next",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -297,7 +297,7 @@ dependencies = [
|
||||
"quote",
|
||||
"strum",
|
||||
"syn 2.0.87",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -343,7 +343,7 @@ dependencies = [
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
@@ -537,7 +537,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
@@ -595,7 +595,7 @@ dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"uuid",
|
||||
]
|
||||
@@ -882,7 +882,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smol_str",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -905,7 +905,7 @@ dependencies = [
|
||||
"serde_with",
|
||||
"smol_str",
|
||||
"stacker",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -921,7 +921,7 @@ dependencies = [
|
||||
"serde_with",
|
||||
"smol_str",
|
||||
"stacker",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"unicode-security",
|
||||
]
|
||||
|
||||
@@ -939,9 +939,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.38"
|
||||
version = "0.4.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
|
||||
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
|
||||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
@@ -1361,39 +1361,6 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
|
||||
|
||||
[[package]]
|
||||
name = "dom_query"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f36ad8fe49e1234652c9e62472a8429c46d03af5e4d230edf499fd352907786"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"foldhash",
|
||||
"html5ever 0.29.0",
|
||||
"precomputed-hash",
|
||||
"selectors",
|
||||
"tendril",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dom_smoothie"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88147dc268d8ede25890504bbbeeb4168d7ea47c1bff921176ddec722660fc1e"
|
||||
dependencies = [
|
||||
"dom_query",
|
||||
"flagset",
|
||||
"gjson",
|
||||
"html-escape",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"tendril",
|
||||
"thiserror 2.0.10",
|
||||
"unicode-segmentation",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "downcast"
|
||||
version = "0.11.0"
|
||||
@@ -1545,12 +1512,6 @@ version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
|
||||
|
||||
[[package]]
|
||||
name = "flagset"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b3ea1ec5f8307826a5b71094dd91fc04d4ae75d5709b20ad351c7fb4815c86ec"
|
||||
|
||||
[[package]]
|
||||
name = "float_next_after"
|
||||
version = "1.0.0"
|
||||
@@ -1563,12 +1524,6 @@ version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
@@ -1600,7 +1555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9"
|
||||
dependencies = [
|
||||
"nonempty",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1854,12 +1809,6 @@ version = "0.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64"
|
||||
|
||||
[[package]]
|
||||
name = "gjson"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43503cc176394dd30a6525f5f36e838339b8b5619be33ed9a7783841580a97b6"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.7"
|
||||
@@ -1986,15 +1935,6 @@ dependencies = [
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-escape"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
|
||||
dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.27.0"
|
||||
@@ -2585,7 +2525,7 @@ dependencies = [
|
||||
"ndarray",
|
||||
"num-traits",
|
||||
"rand",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2711,7 +2651,7 @@ checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e"
|
||||
dependencies = [
|
||||
"miette-derive",
|
||||
"once_cell",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
@@ -3216,7 +3156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"ucd-trie",
|
||||
]
|
||||
|
||||
@@ -3532,7 +3472,7 @@ dependencies = [
|
||||
"rustc-hash 2.0.0",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
@@ -3549,7 +3489,7 @@ dependencies = [
|
||||
"rustc-hash 2.0.0",
|
||||
"rustls",
|
||||
"slab",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
]
|
||||
@@ -3658,7 +3598,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"libredox",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3800,7 +3740,7 @@ dependencies = [
|
||||
"nom",
|
||||
"pin-project-lite",
|
||||
"reqwest",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4304,9 +4244,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.11.0"
|
||||
version = "3.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817"
|
||||
checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
@@ -4322,9 +4262,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.11.0"
|
||||
version = "3.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d"
|
||||
checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
@@ -4401,7 +4341,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-traits",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"time",
|
||||
]
|
||||
|
||||
@@ -4525,7 +4465,7 @@ dependencies = [
|
||||
"byteorder",
|
||||
"memchr",
|
||||
"serde",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4616,7 +4556,7 @@ dependencies = [
|
||||
"serde-content",
|
||||
"serde_json",
|
||||
"surrealdb-core",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tokio-util",
|
||||
@@ -4698,7 +4638,7 @@ dependencies = [
|
||||
"storekey",
|
||||
"subtle",
|
||||
"surrealdb-derive",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"trice",
|
||||
@@ -4856,7 +4796,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"regex",
|
||||
"strum",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
@@ -4866,16 +4806,7 @@ version = "1.0.69"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
|
||||
dependencies = [
|
||||
"thiserror-impl 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3"
|
||||
dependencies = [
|
||||
"thiserror-impl 2.0.10",
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4889,17 +4820,6 @@ dependencies = [
|
||||
"syn 2.0.87",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.87",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.8"
|
||||
@@ -5270,7 +5190,7 @@ dependencies = [
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"url",
|
||||
"utf-8",
|
||||
]
|
||||
@@ -5409,12 +5329,6 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -5793,7 +5707,7 @@ dependencies = [
|
||||
"pharos",
|
||||
"rustc_version",
|
||||
"send_wrapper",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
@@ -5925,8 +5839,8 @@ dependencies = [
|
||||
"axum_session_auth",
|
||||
"axum_session_surreal",
|
||||
"axum_typed_multipart",
|
||||
"chrono",
|
||||
"config",
|
||||
"dom_smoothie",
|
||||
"futures",
|
||||
"lettre",
|
||||
"mime",
|
||||
@@ -5942,7 +5856,7 @@ dependencies = [
|
||||
"surrealdb",
|
||||
"tempfile",
|
||||
"text-splitter",
|
||||
"thiserror 1.0.69",
|
||||
"thiserror",
|
||||
"tiktoken-rs",
|
||||
"tokio",
|
||||
"tower-http",
|
||||
|
||||
@@ -12,8 +12,8 @@ axum_session = "0.14.4"
|
||||
axum_session_auth = "0.14.1"
|
||||
axum_session_surreal = "0.2.1"
|
||||
axum_typed_multipart = "0.12.1"
|
||||
chrono = { version = "0.4.39", features = ["serde"] }
|
||||
config = "0.15.4"
|
||||
dom_smoothie = "0.3.0"
|
||||
futures = "0.3.31"
|
||||
lettre = { version = "0.11.11", features = ["rustls-tls"] }
|
||||
mime = "0.3.17"
|
||||
@@ -46,3 +46,4 @@ path = "src/bin/server.rs"
|
||||
[[bin]]
|
||||
name = "worker"
|
||||
path = "src/bin/worker.rs"
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -20,6 +20,7 @@ need_stdout = false
|
||||
command = [
|
||||
"cargo", "clippy",
|
||||
"--color", "always",
|
||||
"-j","4"
|
||||
]
|
||||
need_stdout = false
|
||||
|
||||
@@ -88,7 +89,7 @@ background = true
|
||||
# Call it as
|
||||
# bacon ex -- my-example
|
||||
[jobs.server]
|
||||
command = ["cargo", "run", "--color", "always", "--bin", "server"]
|
||||
command = ["cargo", "run", "--color", "always", "--bin", "server", "-j", "4"]
|
||||
need_stdout = true
|
||||
allow_warnings = true
|
||||
watch = ["src", "Cargo.toml", "Cargo.lock"]
|
||||
@@ -96,7 +97,7 @@ background = true
|
||||
on_change_strategy = "kill_then_restart"
|
||||
|
||||
[jobs.worker]
|
||||
command = ["cargo", "run", "--color", "always", "--bin", "worker"]
|
||||
command = ["cargo", "run", "--color", "always", "--bin", "worker", "-j", "4"]
|
||||
need_stdout = true
|
||||
allow_warnings = true
|
||||
watch = ["src", "Cargo.toml", "Cargo.lock"]
|
||||
|
||||
@@ -40,7 +40,14 @@ use zettle_db::{
|
||||
},
|
||||
AppState,
|
||||
},
|
||||
storage::{db::SurrealDbClient, types::user::User},
|
||||
storage::{
|
||||
db::SurrealDbClient,
|
||||
types::{
|
||||
file_info::FileInfo, job::Job, knowledge_entity::KnowledgeEntity,
|
||||
knowledge_relationship::KnowledgeRelationship, text_chunk::TextChunk,
|
||||
text_content::TextContent, user::User,
|
||||
},
|
||||
},
|
||||
utils::{config::get_config, mailer::Mailer},
|
||||
};
|
||||
|
||||
@@ -90,8 +97,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
job_queue: Arc::new(JobQueue::new(surreal_db_client)),
|
||||
};
|
||||
|
||||
// setup_auth(&app_state.surreal_db_client).await?;
|
||||
|
||||
let session_config = SessionConfig::default()
|
||||
.with_table_name("test_session_table")
|
||||
.with_secure(true);
|
||||
@@ -104,6 +109,26 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.await?;
|
||||
|
||||
app_state.surreal_db_client.build_indexes().await?;
|
||||
setup_auth(&app_state.surreal_db_client).await?;
|
||||
// app_state.surreal_db_client.drop_table::<FileInfo>().await?;
|
||||
// app_state.surreal_db_client.drop_table::<User>().await?;
|
||||
// app_state.surreal_db_client.drop_table::<Job>().await?;
|
||||
// app_state
|
||||
// .surreal_db_client
|
||||
// .drop_table::<KnowledgeEntity>()
|
||||
// .await?;
|
||||
// app_state
|
||||
// .surreal_db_client
|
||||
// .drop_table::<KnowledgeRelationship>()
|
||||
// .await?;
|
||||
// app_state
|
||||
// .surreal_db_client
|
||||
// .drop_table::<TextContent>()
|
||||
// .await?;
|
||||
// app_state
|
||||
// .surreal_db_client
|
||||
// .drop_table::<TextChunk>()
|
||||
// .await?;
|
||||
|
||||
// Create Axum router
|
||||
let app = Router::new()
|
||||
@@ -177,14 +202,14 @@ fn html_routes(
|
||||
.layer(SessionLayer::new(session_store))
|
||||
}
|
||||
|
||||
// async fn setup_auth(db: &SurrealDbClient) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// db.query(
|
||||
// "DEFINE TABLE user SCHEMALESS;
|
||||
// DEFINE INDEX unique_name ON TABLE user FIELDS email UNIQUE;
|
||||
// DEFINE ACCESS account ON DATABASE TYPE RECORD
|
||||
// SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id)
|
||||
// SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );",
|
||||
// )
|
||||
// .await?;
|
||||
// Ok(())
|
||||
// }
|
||||
async fn setup_auth(db: &SurrealDbClient) -> Result<(), Box<dyn std::error::Error>> {
|
||||
db.query(
|
||||
"DEFINE TABLE user SCHEMALESS;
|
||||
DEFINE INDEX unique_name ON TABLE user FIELDS email UNIQUE;
|
||||
DEFINE ACCESS account ON DATABASE TYPE RECORD
|
||||
SIGNUP ( CREATE user SET email = $email, password = crypto::argon2::generate($password), anonymous = false, user_id = $user_id)
|
||||
SIGNIN ( SELECT * FROM user WHERE email = $email AND crypto::argon2::compare(password, $password) );",
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::task;
|
||||
|
||||
@@ -162,8 +163,11 @@ async fn create_single_entity(
|
||||
|
||||
let embedding = generate_embedding(openai_client, &embedding_input).await?;
|
||||
|
||||
let now = Utc::now();
|
||||
Ok(KnowledgeEntity {
|
||||
id: assigned_id,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
name: llm_entity.name.to_string(),
|
||||
description: llm_entity.description.to_string(),
|
||||
entity_type: KnowledgeEntityType::from(llm_entity.entity_type.to_string()),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use chrono::Utc;
|
||||
use futures::Stream;
|
||||
use std::{
|
||||
sync::Arc,
|
||||
@@ -33,6 +34,7 @@ impl JobQueue {
|
||||
/// Creates a new job and stores it in the database
|
||||
pub async fn enqueue(&self, content: IngressObject, user_id: String) -> Result<Job, AppError> {
|
||||
let job = Job::new(content, user_id).await;
|
||||
info!("{:?}", job);
|
||||
store_item(&self.db, job.clone()).await?;
|
||||
Ok(job)
|
||||
}
|
||||
@@ -46,6 +48,7 @@ impl JobQueue {
|
||||
.await?
|
||||
.take(0)?;
|
||||
|
||||
info!("{:?}", jobs);
|
||||
Ok(jobs)
|
||||
}
|
||||
|
||||
@@ -95,8 +98,11 @@ impl JobQueue {
|
||||
}
|
||||
|
||||
/// Get unfinished jobs, ie newly created and in progress up two times
|
||||
pub async fn get_unfinished_jobs(&self) -> Result<Vec<Job>, AppError> {
|
||||
let jobs: Vec<Job> = self
|
||||
// pub async fn get_unfinished_jobs(&self) -> Result<Vec<Job>, AppError> {
|
||||
pub async fn get_unfinished_jobs(&self) -> Result<(), AppError> {
|
||||
info!("Getting unfinished jobs");
|
||||
// let jobs: Vec<Job> = self
|
||||
let jobs = self
|
||||
.db
|
||||
.query(
|
||||
"SELECT * FROM type::table($table)
|
||||
@@ -110,12 +116,14 @@ impl JobQueue {
|
||||
)
|
||||
.bind(("table", Job::table_name()))
|
||||
.bind(("max_attempts", MAX_ATTEMPTS))
|
||||
.await?
|
||||
.take(0)?;
|
||||
.await?;
|
||||
// .take(0)?;
|
||||
|
||||
println!("Unfinished jobs found: {}", jobs.len());
|
||||
info!("{:?}", jobs);
|
||||
// println!("Unfinished jobs found: {}", jobs.len());
|
||||
|
||||
Ok(jobs)
|
||||
Ok(())
|
||||
// Ok(jobs)
|
||||
}
|
||||
|
||||
// Method to process a single job
|
||||
@@ -124,12 +132,6 @@ impl JobQueue {
|
||||
job: Job,
|
||||
processor: &ContentProcessor,
|
||||
) -> Result<(), AppError> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis()
|
||||
.to_string();
|
||||
|
||||
let current_attempts = match job.status {
|
||||
JobStatus::InProgress { attempts, .. } => attempts + 1,
|
||||
_ => 1,
|
||||
@@ -140,7 +142,7 @@ impl JobQueue {
|
||||
&job.id,
|
||||
JobStatus::InProgress {
|
||||
attempts: current_attempts,
|
||||
last_attempt: now.clone(),
|
||||
last_attempt: Utc::now(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -8,7 +8,7 @@ use axum::{extract::State, http::StatusCode, response::IntoResponse, Extension};
|
||||
use axum_typed_multipart::{FieldData, TryFromMultipart, TypedMultipart};
|
||||
use futures::{future::try_join_all, TryFutureExt};
|
||||
use tempfile::NamedTempFile;
|
||||
use tracing::info;
|
||||
use tracing::{debug, info};
|
||||
|
||||
#[derive(Debug, TryFromMultipart)]
|
||||
pub struct IngressParams {
|
||||
@@ -32,6 +32,8 @@ pub async fn ingress_data(
|
||||
}))
|
||||
.await?;
|
||||
|
||||
debug!("Got file infos");
|
||||
|
||||
let ingress_objects = create_ingress_objects(
|
||||
IngressInput {
|
||||
content: input.content,
|
||||
@@ -41,6 +43,7 @@ pub async fn ingress_data(
|
||||
},
|
||||
user.id.as_str(),
|
||||
)?;
|
||||
debug!("Got ingress objects");
|
||||
|
||||
let futures: Vec<_> = ingress_objects
|
||||
.into_iter()
|
||||
|
||||
@@ -37,6 +37,10 @@ pub async fn index_handler(
|
||||
false => 0,
|
||||
};
|
||||
|
||||
// let latest_text_contents = match auth.current_user.is_some() {
|
||||
// true =>
|
||||
// }
|
||||
|
||||
// let knowledge_entities = User::get_knowledge_entities(
|
||||
// &auth.current_user.clone().unwrap().id,
|
||||
// &state.surreal_db_client,
|
||||
|
||||
@@ -9,6 +9,7 @@ use axum_session_auth::AuthSession;
|
||||
use axum_session_surreal::SessionSurrealPool;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use surrealdb::{engine::any::Any, Surreal};
|
||||
use tracing::info;
|
||||
|
||||
use crate::{
|
||||
error::{AppError, HtmlError},
|
||||
@@ -63,7 +64,8 @@ pub async fn process_signup_and_show_verification(
|
||||
) -> Result<impl IntoResponse, HtmlError> {
|
||||
let user = match User::create_new(form.email, form.password, &state.surreal_db_client).await {
|
||||
Ok(user) => user,
|
||||
Err(_) => {
|
||||
Err(e) => {
|
||||
info!("{:?}", e);
|
||||
return Ok(Html("<p>User already exists</p>").into_response());
|
||||
}
|
||||
};
|
||||
|
||||
@@ -71,9 +71,12 @@ impl FileInfo {
|
||||
let uuid = Uuid::new_v4();
|
||||
let sanitized_file_name = Self::sanitize_file_name(&file_name);
|
||||
|
||||
let now = Utc::now();
|
||||
// Create new FileInfo instance
|
||||
let file_info = Self {
|
||||
id: uuid.to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
sha256,
|
||||
path: Self::persist_file(&uuid, file, &sanitized_file_name, user_id)
|
||||
.await?
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{ingress::types::ingress_object::IngressObject, stored_object};
|
||||
@@ -8,7 +7,7 @@ pub enum JobStatus {
|
||||
Created,
|
||||
InProgress {
|
||||
attempts: u32,
|
||||
last_attempt: String, // timestamp
|
||||
last_attempt: DateTime<Utc>,
|
||||
},
|
||||
Completed,
|
||||
Error(String),
|
||||
@@ -18,24 +17,18 @@ pub enum JobStatus {
|
||||
stored_object!(Job, "job", {
|
||||
content: IngressObject,
|
||||
status: JobStatus,
|
||||
created_at: String,
|
||||
updated_at: String,
|
||||
user_id: String
|
||||
});
|
||||
|
||||
impl Job {
|
||||
pub async fn new(content: IngressObject, user_id: String) -> Self {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis()
|
||||
.to_string();
|
||||
let now = Utc::now();
|
||||
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
content,
|
||||
status: JobStatus::Created,
|
||||
created_at: now.clone(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
user_id,
|
||||
}
|
||||
|
||||
@@ -44,8 +44,11 @@ impl KnowledgeEntity {
|
||||
embedding: Vec<f32>,
|
||||
user_id: String,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
source_id,
|
||||
name,
|
||||
description,
|
||||
|
||||
@@ -18,8 +18,11 @@ impl KnowledgeRelationship {
|
||||
relationship_type: String,
|
||||
metadata: Option<serde_json::Value>,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
in_,
|
||||
out,
|
||||
relationship_type,
|
||||
|
||||
@@ -23,6 +23,7 @@ macro_rules! stored_object {
|
||||
use $crate::storage::types::StoredObject;
|
||||
use serde::de::{self, Visitor};
|
||||
use std::fmt;
|
||||
use chrono::{DateTime, Utc };
|
||||
|
||||
struct FlexibleIdVisitor;
|
||||
|
||||
@@ -64,10 +65,61 @@ macro_rules! stored_object {
|
||||
deserializer.deserialize_any(FlexibleIdVisitor)
|
||||
}
|
||||
|
||||
fn serialize_datetime<S>(date: &DateTime<Utc>, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
Into::<surrealdb::sql::Datetime>::into(*date).serialize(serializer)
|
||||
}
|
||||
|
||||
// fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
|
||||
// where
|
||||
// D: serde::Deserializer<'de>,
|
||||
// {
|
||||
// let dt = surrealdb::sql::Datetime::deserialize(deserializer)?;
|
||||
// Ok(DateTime::<Utc>::from(dt))
|
||||
// }
|
||||
fn deserialize_datetime<'de, D>(deserializer: D) -> Result<DateTime<Utc>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
use serde::de::Error;
|
||||
|
||||
// Accept various formats
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
|
||||
match value {
|
||||
// Handle string format
|
||||
serde_json::Value::String(s) => {
|
||||
if s.starts_with("d\"") && s.ends_with('\"') {
|
||||
let cleaned = &s[2..s.len()-1];
|
||||
DateTime::parse_from_rfc3339(cleaned)
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
.map_err(Error::custom)
|
||||
} else {
|
||||
DateTime::parse_from_rfc3339(&s)
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
.map_err(Error::custom)
|
||||
}
|
||||
},
|
||||
// Handle object format (in case SurrealDB returns datetime as an object)
|
||||
serde_json::Value::Object(_) => {
|
||||
let dt = surrealdb::sql::Datetime::deserialize(value)
|
||||
.map_err(Error::custom)?;
|
||||
Ok(DateTime::<Utc>::from(dt))
|
||||
},
|
||||
_ => Err(Error::custom("unexpected datetime format")),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct $name {
|
||||
#[serde(deserialize_with = "deserialize_flexible_id")]
|
||||
pub id: String,
|
||||
#[serde(serialize_with = "serialize_datetime", deserialize_with = "deserialize_datetime", default)]
|
||||
pub created_at: DateTime<Utc>,
|
||||
#[serde(serialize_with = "serialize_datetime", deserialize_with = "deserialize_datetime", default)]
|
||||
pub updated_at: DateTime<Utc>,
|
||||
$(pub $field: $ty),*
|
||||
}
|
||||
|
||||
|
||||
@@ -10,8 +10,11 @@ stored_object!(TextChunk, "text_chunk", {
|
||||
|
||||
impl TextChunk {
|
||||
pub fn new(source_id: String, chunk: String, embedding: Vec<f32>, user_id: String) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
source_id,
|
||||
chunk,
|
||||
embedding,
|
||||
|
||||
@@ -20,8 +20,11 @@ impl TextContent {
|
||||
file_info: Option<FileInfo>,
|
||||
user_id: String,
|
||||
) -> Self {
|
||||
let now = Utc::now();
|
||||
Self {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
text,
|
||||
file_info,
|
||||
instructions,
|
||||
|
||||
@@ -7,7 +7,7 @@ use axum_session_auth::Authentication;
|
||||
use surrealdb::{engine::any::Any, Surreal};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::knowledge_entity::KnowledgeEntity;
|
||||
use super::{knowledge_entity::KnowledgeEntity, text_content::TextContent};
|
||||
|
||||
stored_object!(User, "user", {
|
||||
email: String,
|
||||
@@ -47,6 +47,8 @@ impl User {
|
||||
return Err(AppError::Auth("User already exists".into()));
|
||||
}
|
||||
|
||||
let now = Utc::now();
|
||||
|
||||
let id = Uuid::new_v4().to_string();
|
||||
let user: Option<User> = db
|
||||
.client
|
||||
@@ -54,11 +56,15 @@ impl User {
|
||||
"CREATE type::thing('user', $id) SET
|
||||
email = $email,
|
||||
password = crypto::argon2::generate($password),
|
||||
anonymous = false",
|
||||
anonymous = false,
|
||||
created_at = $created_at,
|
||||
updated_at = $updated_at",
|
||||
)
|
||||
.bind(("id", id))
|
||||
.bind(("email", email))
|
||||
.bind(("password", password))
|
||||
.bind(("created_at", now))
|
||||
.bind(("updated_at", now))
|
||||
.await?
|
||||
.take(0)?;
|
||||
|
||||
@@ -169,4 +175,18 @@ impl User {
|
||||
|
||||
Ok(entities)
|
||||
}
|
||||
|
||||
pub async fn get_latest_text_contents(
|
||||
id: &str,
|
||||
db: &SurrealDbClient,
|
||||
) -> Result<Vec<TextContent>, AppError> {
|
||||
let items: Vec<TextContent> = db
|
||||
.client
|
||||
.query("SELECT * FROM text_content WHERE user_id = $user_id ORDER BY created_at DESC LIMIT 5")
|
||||
.bind(("user_id", id.to_owned()))
|
||||
.await?
|
||||
.take(0)?;
|
||||
|
||||
Ok(items)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user