mirror of
https://github.com/perstarkse/minne.git
synced 2026-04-22 08:48:30 +02:00
llm analysis wip
This commit is contained in:
Per Stark
1
.gitignore
vendored
1
.gitignore
vendored
@@ -7,3 +7,4 @@
|
|||||||
/target
|
/target
|
||||||
.aider
|
.aider
|
||||||
.aider.chat.history.md .aider.input.history .aider.tags.cache.v3 .aider.tags.cache.v3/cache.db
|
.aider.chat.history.md .aider.input.history .aider.tags.cache.v3 .aider.tags.cache.v3/cache.db
|
||||||
|
data
|
||||||
|
|||||||
373
Cargo.lock
generated
373
Cargo.lock
generated
@@ -163,6 +163,15 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "async-convert"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6d416feee97712e43152cd42874de162b8f9b77295b1c85e5d92725cc8310bae"
|
||||||
|
dependencies = [
|
||||||
|
"async-trait",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-executor"
|
name = "async-executor"
|
||||||
version = "1.13.1"
|
version = "1.13.1"
|
||||||
@@ -261,6 +270,32 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "async-openai"
|
||||||
|
version = "0.24.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c6db3286b4f52b6556ac5208fb575d035eca61a2bf40d7e75d1db2733ffc599f"
|
||||||
|
dependencies = [
|
||||||
|
"async-convert",
|
||||||
|
"backoff",
|
||||||
|
"base64",
|
||||||
|
"bytes",
|
||||||
|
"derive_builder",
|
||||||
|
"eventsource-stream",
|
||||||
|
"futures",
|
||||||
|
"rand",
|
||||||
|
"reqwest",
|
||||||
|
"reqwest-eventsource",
|
||||||
|
"secrecy",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"thiserror",
|
||||||
|
"tokio",
|
||||||
|
"tokio-stream",
|
||||||
|
"tokio-util",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "async-reactor-trait"
|
name = "async-reactor-trait"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
@@ -403,6 +438,20 @@ dependencies = [
|
|||||||
"ubyte",
|
"ubyte",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "backoff"
|
||||||
|
version = "0.4.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"getrandom",
|
||||||
|
"instant",
|
||||||
|
"pin-project-lite",
|
||||||
|
"rand",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "backtrace"
|
name = "backtrace"
|
||||||
version = "0.3.74"
|
version = "0.3.74"
|
||||||
@@ -718,6 +767,37 @@ dependencies = [
|
|||||||
"powerfmt",
|
"powerfmt",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_builder"
|
||||||
|
version = "0.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b"
|
||||||
|
dependencies = [
|
||||||
|
"derive_builder_macro",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_builder_core"
|
||||||
|
version = "0.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38"
|
||||||
|
dependencies = [
|
||||||
|
"darling",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn 2.0.77",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "derive_builder_macro"
|
||||||
|
version = "0.20.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
|
||||||
|
dependencies = [
|
||||||
|
"derive_builder_core",
|
||||||
|
"syn 2.0.77",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "des"
|
name = "des"
|
||||||
version = "0.8.1"
|
version = "0.8.1"
|
||||||
@@ -801,6 +881,17 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "eventsource-stream"
|
||||||
|
version = "0.2.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"nom",
|
||||||
|
"pin-project-lite",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "executor-trait"
|
name = "executor-trait"
|
||||||
version = "2.1.0"
|
version = "2.1.0"
|
||||||
@@ -956,6 +1047,12 @@ version = "0.3.30"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
|
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "futures-timer"
|
||||||
|
version = "3.0.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures-util"
|
name = "futures-util"
|
||||||
version = "0.3.30"
|
version = "0.3.30"
|
||||||
@@ -1097,6 +1194,25 @@ dependencies = [
|
|||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"want",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hyper-rustls"
|
||||||
|
version = "0.27.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
|
||||||
|
dependencies = [
|
||||||
|
"futures-util",
|
||||||
|
"http",
|
||||||
|
"hyper",
|
||||||
|
"hyper-util",
|
||||||
|
"rustls",
|
||||||
|
"rustls-native-certs 0.8.0",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
|
"tower-service",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1106,12 +1222,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
|
checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes",
|
"bytes",
|
||||||
|
"futures-channel",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"hyper",
|
"hyper",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
"socket2 0.5.7",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tower",
|
||||||
|
"tower-service",
|
||||||
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1183,6 +1304,12 @@ dependencies = [
|
|||||||
"windows-sys 0.48.0",
|
"windows-sys 0.48.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ipnet"
|
||||||
|
version = "2.10.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "1.0.11"
|
version = "1.0.11"
|
||||||
@@ -1672,6 +1799,54 @@ dependencies = [
|
|||||||
"unicode-ident",
|
"unicode-ident",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn"
|
||||||
|
version = "0.11.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"pin-project-lite",
|
||||||
|
"quinn-proto",
|
||||||
|
"quinn-udp",
|
||||||
|
"rustc-hash",
|
||||||
|
"rustls",
|
||||||
|
"socket2 0.5.7",
|
||||||
|
"thiserror",
|
||||||
|
"tokio",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn-proto"
|
||||||
|
version = "0.11.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6"
|
||||||
|
dependencies = [
|
||||||
|
"bytes",
|
||||||
|
"rand",
|
||||||
|
"ring",
|
||||||
|
"rustc-hash",
|
||||||
|
"rustls",
|
||||||
|
"slab",
|
||||||
|
"thiserror",
|
||||||
|
"tinyvec",
|
||||||
|
"tracing",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "quinn-udp"
|
||||||
|
version = "0.5.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"once_cell",
|
||||||
|
"socket2 0.5.7",
|
||||||
|
"tracing",
|
||||||
|
"windows-sys 0.59.0",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quote"
|
name = "quote"
|
||||||
version = "1.0.37"
|
version = "1.0.37"
|
||||||
@@ -1807,6 +1982,67 @@ version = "0.8.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "reqwest"
|
||||||
|
version = "0.12.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"bytes",
|
||||||
|
"futures-core",
|
||||||
|
"futures-util",
|
||||||
|
"http",
|
||||||
|
"http-body",
|
||||||
|
"http-body-util",
|
||||||
|
"hyper",
|
||||||
|
"hyper-rustls",
|
||||||
|
"hyper-util",
|
||||||
|
"ipnet",
|
||||||
|
"js-sys",
|
||||||
|
"log",
|
||||||
|
"mime",
|
||||||
|
"mime_guess",
|
||||||
|
"once_cell",
|
||||||
|
"percent-encoding",
|
||||||
|
"pin-project-lite",
|
||||||
|
"quinn",
|
||||||
|
"rustls",
|
||||||
|
"rustls-native-certs 0.8.0",
|
||||||
|
"rustls-pemfile",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"serde",
|
||||||
|
"serde_json",
|
||||||
|
"serde_urlencoded",
|
||||||
|
"sync_wrapper 1.0.1",
|
||||||
|
"tokio",
|
||||||
|
"tokio-rustls",
|
||||||
|
"tokio-util",
|
||||||
|
"tower-service",
|
||||||
|
"url",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"wasm-bindgen-futures",
|
||||||
|
"wasm-streams",
|
||||||
|
"web-sys",
|
||||||
|
"windows-registry",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "reqwest-eventsource"
|
||||||
|
version = "0.6.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde"
|
||||||
|
dependencies = [
|
||||||
|
"eventsource-stream",
|
||||||
|
"futures-core",
|
||||||
|
"futures-timer",
|
||||||
|
"mime",
|
||||||
|
"nom",
|
||||||
|
"pin-project-lite",
|
||||||
|
"reqwest",
|
||||||
|
"thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ring"
|
name = "ring"
|
||||||
version = "0.17.8"
|
version = "0.17.8"
|
||||||
@@ -1828,6 +2064,12 @@ version = "0.1.24"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-hash"
|
||||||
|
version = "2.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rusticata-macros"
|
name = "rusticata-macros"
|
||||||
version = "4.1.0"
|
version = "4.1.0"
|
||||||
@@ -1886,7 +2128,7 @@ checksum = "2a980454b497c439c274f2feae2523ed8138bbd3d323684e1435fec62f800481"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"log",
|
"log",
|
||||||
"rustls",
|
"rustls",
|
||||||
"rustls-native-certs",
|
"rustls-native-certs 0.7.3",
|
||||||
"rustls-pki-types",
|
"rustls-pki-types",
|
||||||
"rustls-webpki",
|
"rustls-webpki",
|
||||||
]
|
]
|
||||||
@@ -1904,6 +2146,19 @@ dependencies = [
|
|||||||
"security-framework",
|
"security-framework",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustls-native-certs"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
|
||||||
|
dependencies = [
|
||||||
|
"openssl-probe",
|
||||||
|
"rustls-pemfile",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"schannel",
|
||||||
|
"security-framework",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-pemfile"
|
name = "rustls-pemfile"
|
||||||
version = "2.1.3"
|
version = "2.1.3"
|
||||||
@@ -1978,6 +2233,16 @@ dependencies = [
|
|||||||
"sha2",
|
"sha2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "secrecy"
|
||||||
|
version = "0.8.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e"
|
||||||
|
dependencies = [
|
||||||
|
"serde",
|
||||||
|
"zeroize",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "security-framework"
|
name = "security-framework"
|
||||||
version = "2.11.1"
|
version = "2.11.1"
|
||||||
@@ -2205,6 +2470,9 @@ name = "sync_wrapper"
|
|||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
|
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "synstructure"
|
name = "synstructure"
|
||||||
@@ -2347,6 +2615,28 @@ dependencies = [
|
|||||||
"syn 2.0.77",
|
"syn 2.0.77",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-rustls"
|
||||||
|
version = "0.26.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
|
||||||
|
dependencies = [
|
||||||
|
"rustls",
|
||||||
|
"rustls-pki-types",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-stream"
|
||||||
|
version = "0.1.16"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
|
||||||
|
dependencies = [
|
||||||
|
"futures-core",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-util"
|
name = "tokio-util"
|
||||||
version = "0.7.12"
|
version = "0.7.12"
|
||||||
@@ -2450,6 +2740,12 @@ dependencies = [
|
|||||||
"tracing-log",
|
"tracing-log",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "try-lock"
|
||||||
|
version = "0.2.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "typenum"
|
name = "typenum"
|
||||||
version = "1.17.0"
|
version = "1.17.0"
|
||||||
@@ -2538,6 +2834,15 @@ version = "1.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
|
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "want"
|
||||||
|
version = "0.3.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
|
||||||
|
dependencies = [
|
||||||
|
"try-lock",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasi"
|
name = "wasi"
|
||||||
version = "0.11.0+wasi-snapshot-preview1"
|
version = "0.11.0+wasi-snapshot-preview1"
|
||||||
@@ -2570,6 +2875,18 @@ dependencies = [
|
|||||||
"wasm-bindgen-shared",
|
"wasm-bindgen-shared",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-bindgen-futures"
|
||||||
|
version = "0.4.43"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"web-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wasm-bindgen-macro"
|
name = "wasm-bindgen-macro"
|
||||||
version = "0.2.93"
|
version = "0.2.93"
|
||||||
@@ -2599,6 +2916,29 @@ version = "0.2.93"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
|
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "wasm-streams"
|
||||||
|
version = "0.4.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd"
|
||||||
|
dependencies = [
|
||||||
|
"futures-util",
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
"wasm-bindgen-futures",
|
||||||
|
"web-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "web-sys"
|
||||||
|
version = "0.3.70"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
|
||||||
|
dependencies = [
|
||||||
|
"js-sys",
|
||||||
|
"wasm-bindgen",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winapi"
|
name = "winapi"
|
||||||
version = "0.3.9"
|
version = "0.3.9"
|
||||||
@@ -2630,6 +2970,36 @@ dependencies = [
|
|||||||
"windows-targets 0.52.6",
|
"windows-targets 0.52.6",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-registry"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
|
||||||
|
dependencies = [
|
||||||
|
"windows-result",
|
||||||
|
"windows-strings",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-result"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
|
||||||
|
dependencies = [
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-strings"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
|
||||||
|
dependencies = [
|
||||||
|
"windows-result",
|
||||||
|
"windows-targets 0.52.6",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "windows-sys"
|
name = "windows-sys"
|
||||||
version = "0.48.0"
|
version = "0.48.0"
|
||||||
@@ -2837,6 +3207,7 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
|
|||||||
name = "zettle_db"
|
name = "zettle_db"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"async-openai",
|
||||||
"axum",
|
"axum",
|
||||||
"axum_typed_multipart",
|
"axum_typed_multipart",
|
||||||
"bytes",
|
"bytes",
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ edition = "2021"
|
|||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
async-openai = "0.24.1"
|
||||||
axum = { version = "0.7.5", features = ["multipart", "macros"] }
|
axum = { version = "0.7.5", features = ["multipart", "macros"] }
|
||||||
axum_typed_multipart = "0.12.1"
|
axum_typed_multipart = "0.12.1"
|
||||||
bytes = { version = "1.7.2", features = ["serde"] }
|
bytes = { version = "1.7.2", features = ["serde"] }
|
||||||
|
|||||||
57
data/182cb7b4-2286-4f40-9237-30b812d67696/flake.nix
Normal file
57
data/182cb7b4-2286-4f40-9237-30b812d67696/flake.nix
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
{
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:cachix/devenv-nixpkgs/rolling";
|
||||||
|
systems.url = "github:nix-systems/default";
|
||||||
|
devenv.url = "github:cachix/devenv";
|
||||||
|
devenv.inputs.nixpkgs.follows = "nixpkgs";
|
||||||
|
};
|
||||||
|
|
||||||
|
nixConfig = {
|
||||||
|
extra-trusted-public-keys = "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=";
|
||||||
|
extra-substituters = "https://devenv.cachix.org";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = {
|
||||||
|
self,
|
||||||
|
nixpkgs,
|
||||||
|
devenv,
|
||||||
|
systems,
|
||||||
|
...
|
||||||
|
} @ inputs: let
|
||||||
|
forEachSystem = nixpkgs.lib.genAttrs (import systems);
|
||||||
|
in {
|
||||||
|
packages = forEachSystem (system: {
|
||||||
|
devenv-up = self.devShells.${system}.default.config.procfileScript;
|
||||||
|
});
|
||||||
|
|
||||||
|
devShells =
|
||||||
|
forEachSystem
|
||||||
|
(system: let
|
||||||
|
pkgs = nixpkgs.legacyPackages.${system};
|
||||||
|
in {
|
||||||
|
default = devenv.lib.mkShell {
|
||||||
|
inherit inputs pkgs;
|
||||||
|
modules = [
|
||||||
|
{
|
||||||
|
# https://devenv.sh/reference/options/
|
||||||
|
enterShell = ''
|
||||||
|
echo "run devenv up -d to start and monitor services"
|
||||||
|
'';
|
||||||
|
|
||||||
|
languages.rust.enable = true;
|
||||||
|
|
||||||
|
services = {
|
||||||
|
redis = {
|
||||||
|
enable = true;
|
||||||
|
};
|
||||||
|
rabbitmq = {
|
||||||
|
enable = true;
|
||||||
|
plugins = ["tracing"];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
use tokio;
|
use tokio;
|
||||||
use tracing::{info, error};
|
use tracing::info;
|
||||||
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
|
||||||
use zettle_db::rabbitmq::{consumer::RabbitMQConsumer, RabbitMQConfig, RabbitMQError};
|
use zettle_db::rabbitmq::{consumer::RabbitMQConsumer, RabbitMQConfig };
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ pub async fn create_ingress_objects(
|
|||||||
input: IngressInput,
|
input: IngressInput,
|
||||||
redis_client: &RedisClient,
|
redis_client: &RedisClient,
|
||||||
) -> Result<Vec<IngressObject>, IngressContentError> {
|
) -> Result<Vec<IngressObject>, IngressContentError> {
|
||||||
|
// Initialize list
|
||||||
let mut object_list = Vec::new();
|
let mut object_list = Vec::new();
|
||||||
|
|
||||||
if let Some(input_content) = input.content {
|
if let Some(input_content) = input.content {
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ impl IngressObject {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Fetches and extracts text from a URL.
|
/// Fetches and extracts text from a URL.
|
||||||
async fn fetch_text_from_url(url: &str) -> Result<String, IngressContentError> {
|
async fn fetch_text_from_url(_url: &str) -> Result<String, IngressContentError> {
|
||||||
unimplemented!()
|
unimplemented!()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,6 +74,10 @@ impl IngressObject {
|
|||||||
// TODO: Implement OCR on image using a crate like `tesseract`
|
// TODO: Implement OCR on image using a crate like `tesseract`
|
||||||
Err(IngressContentError::UnsupportedMime(file_info.mime_type.clone()))
|
Err(IngressContentError::UnsupportedMime(file_info.mime_type.clone()))
|
||||||
}
|
}
|
||||||
|
"application/octet-stream" => {
|
||||||
|
let content = tokio::fs::read_to_string(&file_info.path).await?;
|
||||||
|
Ok(content)
|
||||||
|
}
|
||||||
// Handle other MIME types as needed
|
// Handle other MIME types as needed
|
||||||
_ => Err(IngressContentError::UnsupportedMime(file_info.mime_type.clone())),
|
_ => Err(IngressContentError::UnsupportedMime(file_info.mime_type.clone())),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
|
use async_openai::types::{ ChatCompletionRequestSystemMessage, ChatCompletionRequestUserMessage, CreateChatCompletionRequestArgs};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::json;
|
||||||
|
use tracing::info;
|
||||||
use crate::models::file_info::FileInfo;
|
use crate::models::file_info::FileInfo;
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
@@ -11,6 +14,31 @@ pub struct TextContent {
|
|||||||
pub category: String,
|
pub category: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
pub struct LLMAnalysis {
|
||||||
|
pub json_ld: serde_json::Value,
|
||||||
|
pub description: String,
|
||||||
|
pub related_category: String,
|
||||||
|
pub instructions: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Error types for processing `TextContent`.
|
||||||
|
#[derive(Error, Debug)]
|
||||||
|
pub enum ProcessingError {
|
||||||
|
#[error("LLM processing error: {0}")]
|
||||||
|
LLMError(String),
|
||||||
|
|
||||||
|
#[error("Graph DB storage error: {0}")]
|
||||||
|
GraphDBError(String),
|
||||||
|
|
||||||
|
#[error("Vector DB storage error: {0}")]
|
||||||
|
VectorDBError(String),
|
||||||
|
|
||||||
|
#[error("Unknown processing error")]
|
||||||
|
Unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
impl TextContent {
|
impl TextContent {
|
||||||
/// Creates a new `TextContent` instance.
|
/// Creates a new `TextContent` instance.
|
||||||
pub fn new(text: String, file_info: Option<FileInfo>, instructions: String, category: String) -> Self {
|
pub fn new(text: String, file_info: Option<FileInfo>, instructions: String, category: String) -> Self {
|
||||||
@@ -26,42 +54,101 @@ impl TextContent {
|
|||||||
pub async fn process(&self) -> Result<(), ProcessingError> {
|
pub async fn process(&self) -> Result<(), ProcessingError> {
|
||||||
// Step 1: Send to LLM for analysis
|
// Step 1: Send to LLM for analysis
|
||||||
let analysis = self.send_to_llm().await?;
|
let analysis = self.send_to_llm().await?;
|
||||||
|
info!("{:?}", analysis);
|
||||||
|
|
||||||
// Step 2: Store analysis results in Graph DB
|
// Step 2: Store analysis results in Graph DB
|
||||||
self.store_in_graph_db(&analysis).await?;
|
// self.store_in_graph_db(&analysis).await?;
|
||||||
|
|
||||||
// Step 3: Split text and store in Vector DB
|
// Step 3: Split text and store in Vector DB
|
||||||
self.store_in_vector_db().await?;
|
// self.store_in_vector_db().await?;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sends text to an LLM for analysis.
|
/// Sends text to an LLM for analysis.
|
||||||
async fn send_to_llm(&self) -> Result<LLMAnalysis, ProcessingError> {
|
async fn send_to_llm(&self) -> Result<LLMAnalysis, ProcessingError> {
|
||||||
// TODO: Implement interaction with your specific LLM API.
|
let client = async_openai::Client::new();
|
||||||
// Example using reqwest:
|
|
||||||
/*
|
// Define the JSON Schema for the expected response
|
||||||
let client = reqwest::Client::new();
|
let schema = json!({
|
||||||
let response = client.post("http://llm-api/analyze")
|
"type": "object",
|
||||||
.json(&serde_json::json!({ "text": self.text }))
|
"properties": {
|
||||||
.send()
|
"json_ld": {
|
||||||
.await
|
"type": "object",
|
||||||
.map_err(|e| ProcessingError::LLMError(e.to_string()))?;
|
"properties": {
|
||||||
|
"@context": { "type": "string" },
|
||||||
if !response.status().is_success() {
|
"@type": { "type": "string" },
|
||||||
return Err(ProcessingError::LLMError(format!("LLM API returned status: {}", response.status())));
|
"name": { "type": "string" }
|
||||||
|
// Define only the essential properties
|
||||||
|
},
|
||||||
|
"required": ["@context", "@type", "name"],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
"description": { "type": "string" },
|
||||||
|
"related_category": { "type": "string" },
|
||||||
|
"instructions": { "type": "string" }
|
||||||
|
},
|
||||||
|
"required": ["json_ld", "description", "related_category", "instructions"],
|
||||||
|
"additionalProperties": false
|
||||||
|
});
|
||||||
|
|
||||||
|
let response_format = async_openai::types::ResponseFormat::JsonSchema {
|
||||||
|
json_schema: async_openai::types::ResponseFormatJsonSchema {
|
||||||
|
description: Some("Structured analysis of the submitted content".into()),
|
||||||
|
name: "content_analysis".into(),
|
||||||
|
schema: Some(schema),
|
||||||
|
strict: Some(true),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// Construct the system and user messages
|
||||||
|
let system_message = format!(
|
||||||
|
"You are an expert document analyzer. You will receive a document's text content, along with user instructions and a category. Your task is to provide a structured JSON-LD object representing the content, a short description of the document, how it relates to the submitted category, and any relevant instructions."
|
||||||
|
);
|
||||||
|
|
||||||
|
let user_message = format!(
|
||||||
|
"Category: {}\nInstructions: {}\nContent:\n{}",
|
||||||
|
self.category, self.instructions, self.text
|
||||||
|
);
|
||||||
|
|
||||||
|
// Build the chat completion request
|
||||||
|
let request = CreateChatCompletionRequestArgs::default()
|
||||||
|
.model("gpt-4o-mini")
|
||||||
|
.max_tokens(1024u32)
|
||||||
|
.messages([
|
||||||
|
ChatCompletionRequestSystemMessage::from(system_message).into(),
|
||||||
|
ChatCompletionRequestUserMessage::from(user_message).into(),
|
||||||
|
])
|
||||||
|
.response_format(response_format)
|
||||||
|
.build().map_err(|e| ProcessingError::LLMError(e.to_string()))?;
|
||||||
|
|
||||||
|
// Send the request to OpenAI
|
||||||
|
let response = client.chat().create(request).await.map_err(|e| {
|
||||||
|
ProcessingError::LLMError(format!("OpenAI API request failed: {}", e.to_string()))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
info!("{:?}", response);
|
||||||
|
|
||||||
|
// Extract and parse the response
|
||||||
|
for choice in response.choices {
|
||||||
|
if let Some(content) = choice.message.content {
|
||||||
|
let analysis: LLMAnalysis = serde_json::from_str(&content).map_err(|e| {
|
||||||
|
ProcessingError::LLMError(format!(
|
||||||
|
"Failed to parse LLM response into LLMAnalysis: {}",
|
||||||
|
e.to_string()
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
return Ok(analysis);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let analysis: LLMAnalysis = response.json().await
|
Err(ProcessingError::LLMError(
|
||||||
.map_err(|e| ProcessingError::LLMError(e.to_string()))?;
|
"No content found in LLM response".into(),
|
||||||
|
))
|
||||||
Ok(analysis)
|
|
||||||
*/
|
|
||||||
unimplemented!()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stores analysis results in a graph database.
|
/// Stores analysis results in a graph database.
|
||||||
async fn store_in_graph_db(&self, analysis: &LLMAnalysis) -> Result<(), ProcessingError> {
|
async fn store_in_graph_db(&self, _analysis: &LLMAnalysis) -> Result<(), ProcessingError> {
|
||||||
// TODO: Implement storage logic for your specific graph database.
|
// TODO: Implement storage logic for your specific graph database.
|
||||||
// Example:
|
// Example:
|
||||||
/*
|
/*
|
||||||
@@ -85,28 +172,3 @@ impl TextContent {
|
|||||||
unimplemented!()
|
unimplemented!()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Represents the analysis results from the LLM.
|
|
||||||
#[derive(Debug, Serialize, Deserialize, Clone)]
|
|
||||||
pub struct LLMAnalysis {
|
|
||||||
pub entities: Vec<String>,
|
|
||||||
pub summary: String,
|
|
||||||
// Add other fields based on your LLM's output.
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Error types for processing `TextContent`.
|
|
||||||
#[derive(Error, Debug)]
|
|
||||||
pub enum ProcessingError {
|
|
||||||
#[error("LLM processing error: {0}")]
|
|
||||||
LLMError(String),
|
|
||||||
|
|
||||||
#[error("Graph DB storage error: {0}")]
|
|
||||||
GraphDBError(String),
|
|
||||||
|
|
||||||
#[error("Vector DB storage error: {0}")]
|
|
||||||
VectorDBError(String),
|
|
||||||
|
|
||||||
#[error("Unknown processing error")]
|
|
||||||
Unknown,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ use lapin::{
|
|||||||
};
|
};
|
||||||
use futures_lite::stream::StreamExt;
|
use futures_lite::stream::StreamExt;
|
||||||
|
|
||||||
use crate::models::{ingress_content::IngressContentError, ingress_object::IngressObject};
|
use crate::models::{ingress_content::IngressContentError, ingress_object::IngressObject, text_content::TextContent};
|
||||||
|
|
||||||
use super::{RabbitMQCommon, RabbitMQConfig, RabbitMQError};
|
use super::{RabbitMQCommon, RabbitMQConfig, RabbitMQError};
|
||||||
use tracing::{info, error};
|
use tracing::{info, error};
|
||||||
@@ -108,7 +108,9 @@ impl RabbitMQConsumer {
|
|||||||
match self.consume().await {
|
match self.consume().await {
|
||||||
Ok((ingress, delivery)) => {
|
Ok((ingress, delivery)) => {
|
||||||
info!("Received IngressObject: {:?}", ingress);
|
info!("Received IngressObject: {:?}", ingress);
|
||||||
|
let text_content = ingress.to_text_content().await.unwrap();
|
||||||
|
text_content.process().await.unwrap();
|
||||||
|
|
||||||
self.ack_delivery(delivery).await?;
|
self.ack_delivery(delivery).await?;
|
||||||
// Process the IngressContent
|
// Process the IngressContent
|
||||||
// match self.handle_ingress_content(&ingress).await {
|
// match self.handle_ingress_content(&ingress).await {
|
||||||
@@ -143,109 +145,9 @@ impl RabbitMQConsumer {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
// /// Processes messages in a loop
|
pub async fn handle_ingress_content(&self, ingress: &IngressObject) -> Result<(), IngressContentError> {
|
||||||
// pub async fn process_messages(&self) -> Result<(), RabbitMQError> {
|
|
||||||
// loop {
|
|
||||||
// match self.consume().await {
|
|
||||||
// Ok((ingress, delivery)) => {
|
|
||||||
// // info!("Received ingress object: {:?}", ingress);
|
|
||||||
|
|
||||||
// // Process the ingress object
|
|
||||||
// self.handle_ingress_content(&ingress).await;
|
|
||||||
|
|
||||||
// info!("Processing done, acknowledging message");
|
|
||||||
// self.ack_delivery(delivery).await?;
|
|
||||||
// }
|
|
||||||
// Err(RabbitMQError::ConsumeError(e)) => {
|
|
||||||
// error!("Error consuming message: {}", e);
|
|
||||||
// // Optionally add a delay before trying again
|
|
||||||
// tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
|
|
||||||
// }
|
|
||||||
// Err(e) => {
|
|
||||||
// error!("Unexpected error: {}", e);
|
|
||||||
// break;
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Ok(())
|
|
||||||
// }
|
|
||||||
pub async fn handle_ingress_content(&self, ingress: &IngressObject) -> Result<(), IngressContentError> {
|
|
||||||
info!("Processing IngressContent: {:?}", ingress);
|
info!("Processing IngressContent: {:?}", ingress);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// Convert IngressContent to individual TextContent instances
|
|
||||||
// let text_contents = ingress.to_text_contents().await?;
|
|
||||||
// info!("Generated {} TextContent instances", text_contents.len());
|
|
||||||
|
|
||||||
// // Limit concurrent processing (e.g., 10 at a time)
|
|
||||||
// let semaphore = Arc::new(Semaphore::new(10));
|
|
||||||
// let mut processing_futures = FuturesUnordered::new();
|
|
||||||
|
|
||||||
// for text_content in text_contents {
|
|
||||||
// let semaphore_clone = semaphore.clone();
|
|
||||||
// processing_futures.push(tokio::spawn(async move {
|
|
||||||
// let _permit = semaphore_clone.acquire().await;
|
|
||||||
// match text_content.process().await {
|
|
||||||
// Ok(_) => {
|
|
||||||
// info!("Successfully processed TextContent");
|
|
||||||
// Ok(())
|
|
||||||
// }
|
|
||||||
// Err(e) => {
|
|
||||||
// error!("Error processing TextContent: {:?}", e);
|
|
||||||
// Err(e)
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }));
|
|
||||||
// }
|
|
||||||
|
|
||||||
// // Await all processing tasks
|
|
||||||
// while let Some(result) = processing_futures.next().await {
|
|
||||||
// match result {
|
|
||||||
// Ok(Ok(_)) => {
|
|
||||||
// // Successfully processed
|
|
||||||
// }
|
|
||||||
// Ok(Err(e)) => {
|
|
||||||
// // Processing failed, already logged
|
|
||||||
// }
|
|
||||||
// Err(e) => {
|
|
||||||
// // Task join error
|
|
||||||
// error!("Task join error: {:?}", e);
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
// Ok(())
|
|
||||||
unimplemented!()
|
unimplemented!()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
// /// Handles the IngressContent based on its type
|
|
||||||
// async fn handle_ingress_content(&self, ingress: &IngressContent) {
|
|
||||||
// info!("Processing content: {:?}", ingress);
|
|
||||||
// // There are three different situations:
|
|
||||||
// // 1. There is no ingress.content but there are one or more files
|
|
||||||
// // - We should process the files and act based upon the mime type
|
|
||||||
// // - All different kinds of content return text
|
|
||||||
// // 2. There is ingress.content but there are no files
|
|
||||||
// // - We process ingress.content differently if its a URL or Text enum
|
|
||||||
// // - Return text
|
|
||||||
// // 3. There is ingress.content and files
|
|
||||||
// // - We do both
|
|
||||||
// //
|
|
||||||
// // At the end of processing we have one or several text objects with some associated
|
|
||||||
// // metadata, such as the FileInfo metadata, or the Url associated to the text
|
|
||||||
// //
|
|
||||||
// // There will always be ingress.instructions and ingress.category
|
|
||||||
// //
|
|
||||||
// // When we have all the text objects and metadata, we can begin the next processing
|
|
||||||
// // Here we will:
|
|
||||||
// // 1. Send the text content and metadata to a LLM for analyzing
|
|
||||||
// // - We want several things, JSON_LD metadata, possibly actions
|
|
||||||
// // 2. Store the JSON_LD in a graph database
|
|
||||||
// // 3. Split up the text intelligently and store it in a vector database
|
|
||||||
// //
|
|
||||||
// // We return the function if all succeeds.
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
// pub mod mime;
|
// pub mod mime;
|
||||||
|
// pub mod llm;
|
||||||
|
|||||||
Reference in New Issue
Block a user