mirror of
https://github.com/perstarkse/minne.git
synced 2026-06-26 11:56:19 +02:00
release: 1.0.5
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "evaluations"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
+45
-46
@@ -3,7 +3,7 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use clap::{Args, Parser, ValueEnum};
|
||||
|
||||
use crate::datasets::DatasetKind;
|
||||
@@ -394,26 +394,26 @@ impl Config {
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(k) = self.retrieval.chunk_rrf_k {
|
||||
if k <= 0.0 || !k.is_finite() {
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-k must be a positive, finite number (got {k})"
|
||||
));
|
||||
}
|
||||
if let Some(k) = self.retrieval.chunk_rrf_k
|
||||
&& (k <= 0.0 || !k.is_finite())
|
||||
{
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-k must be a positive, finite number (got {k})"
|
||||
));
|
||||
}
|
||||
if let Some(weight) = self.retrieval.chunk_rrf_vector_weight {
|
||||
if weight < 0.0 || !weight.is_finite() {
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-vector-weight must be a non-negative, finite number (got {weight})"
|
||||
));
|
||||
}
|
||||
if let Some(weight) = self.retrieval.chunk_rrf_vector_weight
|
||||
&& (weight < 0.0 || !weight.is_finite())
|
||||
{
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-vector-weight must be a non-negative, finite number (got {weight})"
|
||||
));
|
||||
}
|
||||
if let Some(weight) = self.retrieval.chunk_rrf_fts_weight {
|
||||
if weight < 0.0 || !weight.is_finite() {
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-fts-weight must be a non-negative, finite number (got {weight})"
|
||||
));
|
||||
}
|
||||
if let Some(weight) = self.retrieval.chunk_rrf_fts_weight
|
||||
&& (weight < 0.0 || !weight.is_finite())
|
||||
{
|
||||
return Err(anyhow!(
|
||||
"--chunk-rrf-fts-weight must be a non-negative, finite number (got {weight})"
|
||||
));
|
||||
}
|
||||
|
||||
if self.concurrency == 0 {
|
||||
@@ -426,16 +426,16 @@ impl Config {
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(query_model) = &self.query_model {
|
||||
if query_model.trim().is_empty() {
|
||||
return Err(anyhow!("--query-model requires a non-empty model name"));
|
||||
}
|
||||
if let Some(query_model) = &self.query_model
|
||||
&& query_model.trim().is_empty()
|
||||
{
|
||||
return Err(anyhow!("--query-model requires a non-empty model name"));
|
||||
}
|
||||
|
||||
if let Some(grow) = self.slice_grow {
|
||||
if grow == 0 {
|
||||
return Err(anyhow!("--slice-grow must be greater than zero"));
|
||||
}
|
||||
if let Some(grow) = self.slice_grow
|
||||
&& grow == 0
|
||||
{
|
||||
return Err(anyhow!("--slice-grow must be greater than zero"));
|
||||
}
|
||||
|
||||
if self.negative_multiplier <= 0.0 || !self.negative_multiplier.is_finite() {
|
||||
@@ -465,12 +465,11 @@ impl Config {
|
||||
}
|
||||
|
||||
// Handle perf log dir env var fallback
|
||||
if self.perf_log_dir.is_none() {
|
||||
if let Ok(dir) = env::var("EVAL_PERF_LOG_DIR") {
|
||||
if !dir.trim().is_empty() {
|
||||
self.perf_log_dir = Some(PathBuf::from(dir));
|
||||
}
|
||||
}
|
||||
if self.perf_log_dir.is_none()
|
||||
&& let Ok(dir) = env::var("EVAL_PERF_LOG_DIR")
|
||||
&& !dir.trim().is_empty()
|
||||
{
|
||||
self.perf_log_dir = Some(PathBuf::from(dir));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -480,10 +479,10 @@ impl Config {
|
||||
let catalog = crate::datasets::catalog()?;
|
||||
let entry = catalog.dataset(self.dataset.id())?;
|
||||
|
||||
if self.slice.is_none() {
|
||||
if let Some(default_slice) = entry.slices.first() {
|
||||
self.slice = Some(default_slice.id.clone());
|
||||
}
|
||||
if self.slice.is_none()
|
||||
&& let Some(default_slice) = entry.slices.first()
|
||||
{
|
||||
self.slice = Some(default_slice.id.clone());
|
||||
}
|
||||
|
||||
let Some(slice_id) = self.slice.as_deref() else {
|
||||
@@ -498,11 +497,11 @@ impl Config {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(limit) = slice.limit {
|
||||
if self.limit_arg == 200 {
|
||||
self.limit_arg = limit;
|
||||
self.limit = Some(limit);
|
||||
}
|
||||
if let Some(limit) = slice.limit
|
||||
&& self.limit_arg == 200
|
||||
{
|
||||
self.limit_arg = limit;
|
||||
self.limit = Some(limit);
|
||||
}
|
||||
if self.corpus_limit.is_none() {
|
||||
self.corpus_limit = slice.corpus_limit;
|
||||
@@ -514,10 +513,10 @@ impl Config {
|
||||
self.llm_mode = include_unanswerable;
|
||||
self.retrieval.require_verified_chunks = !include_unanswerable;
|
||||
}
|
||||
if let Some(multiplier) = slice.negative_multiplier {
|
||||
if negative_multiplier_is_default(self.negative_multiplier) {
|
||||
self.negative_multiplier = multiplier;
|
||||
}
|
||||
if let Some(multiplier) = slice.negative_multiplier
|
||||
&& negative_multiplier_is_default(self.negative_multiplier)
|
||||
{
|
||||
self.negative_multiplier = multiplier;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ use crate::{
|
||||
args::Config,
|
||||
corpus::{self, CorpusCacheConfig},
|
||||
datasets::{
|
||||
beir_subset_store_summary, beir_subset_stores_ready, content_checksum_for_layout,
|
||||
detect_layout, mix_content_checksum, store_dir_for, ConvertedLayout, DatasetKind,
|
||||
ConvertedLayout, DatasetKind, beir_subset_store_summary, beir_subset_stores_ready,
|
||||
content_checksum_for_layout, detect_layout, mix_content_checksum, store_dir_for,
|
||||
},
|
||||
db::{connect_eval_db, default_database, default_namespace, namespace_has_corpus},
|
||||
slice::{self, ledger_target},
|
||||
|
||||
@@ -8,8 +8,9 @@ pub use orchestrator::{
|
||||
load_cached_manifest, persist_corpus_manifest,
|
||||
};
|
||||
pub use store::{
|
||||
seed_manifest_into_db, window_manifest, CorpusHandle, CorpusManifest, CorpusMetadata,
|
||||
CorpusQuestion, NamespaceSeedRecord, ParagraphShard, ParagraphShardStore, MANIFEST_VERSION,
|
||||
CorpusHandle, CorpusManifest, CorpusMetadata, CorpusQuestion, MANIFEST_VERSION,
|
||||
NamespaceSeedRecord, ParagraphShard, ParagraphShardStore, seed_manifest_into_db,
|
||||
window_manifest,
|
||||
};
|
||||
|
||||
pub fn make_ingestion_config(config: &crate::args::Config) -> ingestion_pipeline::IngestionConfig {
|
||||
|
||||
@@ -6,14 +6,14 @@ use std::{
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_openai::Client;
|
||||
use chrono::Utc;
|
||||
use common::{
|
||||
storage::{
|
||||
db::SurrealDbClient,
|
||||
store::{DynStorage, StorageManager},
|
||||
types::{ingestion_payload::IngestionPayload, ingestion_task::IngestionTask, StoredObject},
|
||||
types::{StoredObject, ingestion_payload::IngestionPayload, ingestion_task::IngestionTask},
|
||||
},
|
||||
utils::config::{AppConfig, StorageKind},
|
||||
};
|
||||
@@ -31,7 +31,7 @@ use crate::{
|
||||
|
||||
use crate::corpus::{
|
||||
CorpusCacheConfig, CorpusHandle, CorpusManifest, CorpusMetadata, CorpusQuestion,
|
||||
ParagraphShard, ParagraphShardStore, MANIFEST_VERSION,
|
||||
MANIFEST_VERSION, ParagraphShard, ParagraphShardStore,
|
||||
};
|
||||
|
||||
const INGESTION_SPEC_VERSION: u32 = 2;
|
||||
|
||||
@@ -5,16 +5,17 @@ use std::{
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use chrono::{DateTime, Utc};
|
||||
use common::storage::{
|
||||
db::SurrealDbClient,
|
||||
types::{
|
||||
knowledge_entity::KnowledgeEntity, knowledge_relationship::KnowledgeRelationship,
|
||||
text_chunk::TextChunk, text_content::TextContent, StoredObject,
|
||||
StoredObject, knowledge_entity::KnowledgeEntity,
|
||||
knowledge_relationship::KnowledgeRelationship, text_chunk::TextChunk,
|
||||
text_content::TextContent,
|
||||
},
|
||||
};
|
||||
use ingestion_pipeline::{persist_artifacts, IngestionTuning, PipelineArtifacts};
|
||||
use ingestion_pipeline::{IngestionTuning, PipelineArtifacts, persist_artifacts};
|
||||
use serde::Deserialize;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
@@ -304,7 +305,7 @@ impl ParagraphShardStore {
|
||||
Ok(file) => file,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(err) => {
|
||||
return Err(err).with_context(|| format!("opening shard {}", path.display()))
|
||||
return Err(err).with_context(|| format!("opening shard {}", path.display()));
|
||||
}
|
||||
};
|
||||
let reader = BufReader::new(file);
|
||||
|
||||
@@ -5,7 +5,7 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use serde::Deserialize;
|
||||
use tracing::warn;
|
||||
|
||||
@@ -138,10 +138,10 @@ pub fn convert_beir_documents(
|
||||
continue;
|
||||
};
|
||||
|
||||
if let Some(filter) = doc_ids {
|
||||
if !filter.contains(&best.doc_id) {
|
||||
continue;
|
||||
}
|
||||
if let Some(filter) = doc_ids
|
||||
&& !filter.contains(&best.doc_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(¶graph_slot) = paragraph_index.get(&best.doc_id) else {
|
||||
|
||||
@@ -1,17 +1,16 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::info;
|
||||
|
||||
use super::{
|
||||
beir,
|
||||
BEIR_DATASETS, ConvertedDataset, DatasetKind, DatasetMetadata, beir,
|
||||
checksum::hash_file,
|
||||
store::{
|
||||
self, build_dataset_from_catalog, paragraph_path, read_meta, store_dir_for,
|
||||
upsert_sharded_paragraphs, write_sharded,
|
||||
},
|
||||
ConvertedDataset, DatasetKind, DatasetMetadata, BEIR_DATASETS,
|
||||
};
|
||||
use crate::{args::Config, slice};
|
||||
|
||||
|
||||
@@ -112,10 +112,10 @@ pub fn write_sidecar(content_path: &Path, sha256: &str) -> Result<()> {
|
||||
#[cfg(test)]
|
||||
pub fn content_checksum(content_path: &Path) -> Result<String> {
|
||||
let sidecar_path = ChecksumSidecar::sidecar_path(content_path);
|
||||
if let Some(sidecar) = read_sidecar(&sidecar_path)? {
|
||||
if sidecar.is_valid_for(content_path) {
|
||||
return Ok(sidecar.sha256);
|
||||
}
|
||||
if let Some(sidecar) = read_sidecar(&sidecar_path)?
|
||||
&& sidecar.is_valid_for(content_path)
|
||||
{
|
||||
return Ok(sidecar.sha256);
|
||||
}
|
||||
let sha256 = hash_file(content_path)?;
|
||||
write_sidecar(content_path, &sha256)?;
|
||||
@@ -125,19 +125,17 @@ pub fn content_checksum(content_path: &Path) -> Result<String> {
|
||||
pub fn store_aggregate_checksum(store_dir: &Path) -> Result<String> {
|
||||
let marker = store_dir.join("checksum.sha256");
|
||||
let meta = store_dir.join("meta.json");
|
||||
if marker.is_file() && meta.is_file() {
|
||||
if let (Ok(marker_meta), Ok(meta_meta)) = (marker.metadata(), meta.metadata()) {
|
||||
if marker_meta
|
||||
.modified()
|
||||
.ok()
|
||||
.zip(meta_meta.modified().ok())
|
||||
.is_some_and(|(marker_modified, meta_modified)| marker_modified >= meta_modified)
|
||||
{
|
||||
if let Some(sidecar) = read_sidecar(&marker)? {
|
||||
return Ok(sidecar.sha256);
|
||||
}
|
||||
}
|
||||
}
|
||||
if marker.is_file()
|
||||
&& meta.is_file()
|
||||
&& let (Ok(marker_meta), Ok(meta_meta)) = (marker.metadata(), meta.metadata())
|
||||
&& marker_meta
|
||||
.modified()
|
||||
.ok()
|
||||
.zip(meta_meta.modified().ok())
|
||||
.is_some_and(|(marker_modified, meta_modified)| marker_modified >= meta_modified)
|
||||
&& let Some(sidecar) = read_sidecar(&marker)?
|
||||
{
|
||||
return Ok(sidecar.sha256);
|
||||
}
|
||||
|
||||
let mut entries = Vec::new();
|
||||
|
||||
@@ -4,12 +4,11 @@ use anyhow::{Context, Result};
|
||||
use tracing::info;
|
||||
|
||||
use super::{
|
||||
catalog,
|
||||
ConvertedDataset, DatasetKind, catalog,
|
||||
store::{
|
||||
self, build_dataset_from_catalog, detect_layout, read_meta, store_dir_for, write_sharded,
|
||||
ConvertedLayout,
|
||||
self, ConvertedLayout, build_dataset_from_catalog, detect_layout, read_meta, store_dir_for,
|
||||
write_sharded,
|
||||
},
|
||||
ConvertedDataset, DatasetKind,
|
||||
};
|
||||
use crate::{
|
||||
args::Config,
|
||||
@@ -69,21 +68,19 @@ fn load_from_store(
|
||||
let meta = read_meta(store_dir)?;
|
||||
validate_metadata_fields(&meta.metadata, dataset_kind, config)?;
|
||||
|
||||
if allow_partial {
|
||||
if let Some(paragraph_ids) = slice_paragraph_ids_for_fast_path(config)? {
|
||||
let unique: HashSet<String> = paragraph_ids.into_iter().collect();
|
||||
info!(
|
||||
paragraphs = unique.len(),
|
||||
store = %store_dir.display(),
|
||||
"Loading slice-addressed paragraphs from sharded converted store"
|
||||
);
|
||||
let dataset = build_dataset_from_catalog(store_dir, &unique)?;
|
||||
return Ok(LoadedDataset {
|
||||
dataset,
|
||||
content_checksum: checksum,
|
||||
partial: true,
|
||||
});
|
||||
}
|
||||
if allow_partial && let Some(paragraph_ids) = slice_paragraph_ids_for_fast_path(config)? {
|
||||
let unique: HashSet<String> = paragraph_ids.into_iter().collect();
|
||||
info!(
|
||||
paragraphs = unique.len(),
|
||||
store = %store_dir.display(),
|
||||
"Loading slice-addressed paragraphs from sharded converted store"
|
||||
);
|
||||
let dataset = build_dataset_from_catalog(store_dir, &unique)?;
|
||||
return Ok(LoadedDataset {
|
||||
dataset,
|
||||
content_checksum: checksum,
|
||||
partial: true,
|
||||
});
|
||||
}
|
||||
|
||||
info!(
|
||||
|
||||
@@ -13,7 +13,7 @@ use std::{
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use clap::ValueEnum;
|
||||
use once_cell::sync::OnceCell;
|
||||
@@ -226,7 +226,7 @@ pub use beir_mix::{beir_subset_store_summary, beir_subset_stores_ready, mix_cont
|
||||
pub use checksum::store_aggregate_checksum;
|
||||
pub use loader::{prebuild_catalog_slices, prepare_dataset};
|
||||
pub use store::{
|
||||
content_checksum_for_layout, detect_layout, store_dir_for, write_sharded, ConvertedLayout,
|
||||
ConvertedLayout, content_checksum_for_layout, detect_layout, store_dir_for, write_sharded,
|
||||
};
|
||||
|
||||
pub fn catalog() -> Result<&'static DatasetCatalog> {
|
||||
@@ -383,7 +383,9 @@ impl FromStr for DatasetKind {
|
||||
"scifact" => Ok(Self::Scifact),
|
||||
"nq-beir" | "natural-questions-beir" => Ok(Self::NqBeir),
|
||||
other => {
|
||||
anyhow::bail!("unknown dataset '{other}'. Expected one of: squad, natural-questions, beir, fever, fiqa, hotpotqa, nfcorpus, quora, trec-covid, scifact, nq-beir.")
|
||||
anyhow::bail!(
|
||||
"unknown dataset '{other}'. Expected one of: squad, natural-questions, beir, fever, fiqa, hotpotqa, nfcorpus, quora, trec-covid, scifact, nq-beir."
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,14 +5,14 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::info;
|
||||
|
||||
use super::{
|
||||
checksum::store_aggregate_checksum, ConvertedDataset, ConvertedParagraph, ConvertedQuestion,
|
||||
DatasetMetadata,
|
||||
ConvertedDataset, ConvertedParagraph, ConvertedQuestion, DatasetMetadata,
|
||||
checksum::store_aggregate_checksum,
|
||||
};
|
||||
use crate::slice;
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use chrono::Utc;
|
||||
use common::{
|
||||
storage::{
|
||||
db::SurrealDbClient,
|
||||
types::user::{Theme, User},
|
||||
types::StoredObject,
|
||||
types::user::{Theme, User},
|
||||
},
|
||||
utils::embedding::EmbeddingProvider,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{collections::HashMap, fs, path::Path};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use common::storage::{db::SurrealDbClient, types::text_chunk::TextChunk};
|
||||
|
||||
use crate::{args::Config, corpus, db::connect_eval_db};
|
||||
|
||||
+15
-16
@@ -17,36 +17,35 @@ mod types;
|
||||
use anyhow::Context;
|
||||
use tokio::runtime::Builder;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::{fmt, EnvFilter};
|
||||
use tracing_subscriber::{EnvFilter, fmt};
|
||||
|
||||
/// Configure `SurrealDB` environment variables for optimal performance
|
||||
#[allow(clippy::arithmetic_side_effects, clippy::unwrap_used)]
|
||||
fn configure_surrealdb_performance(cpu_count: usize) {
|
||||
let indexing_batch_size = std::env::var("SURREAL_INDEXING_BATCH_SIZE")
|
||||
.unwrap_or_else(|_| (cpu_count * 2).to_string());
|
||||
std::env::set_var("SURREAL_INDEXING_BATCH_SIZE", indexing_batch_size);
|
||||
|
||||
let max_order_queue = std::env::var("SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE")
|
||||
.unwrap_or_else(|_| (cpu_count * 4).to_string());
|
||||
std::env::set_var(
|
||||
"SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE",
|
||||
max_order_queue,
|
||||
);
|
||||
|
||||
let websocket_concurrent = std::env::var("SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS")
|
||||
.unwrap_or_else(|_| cpu_count.to_string());
|
||||
std::env::set_var(
|
||||
"SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS",
|
||||
websocket_concurrent,
|
||||
);
|
||||
|
||||
let websocket_buffer = std::env::var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE")
|
||||
.unwrap_or_else(|_| (cpu_count * 8).to_string());
|
||||
std::env::set_var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE", websocket_buffer);
|
||||
|
||||
let transaction_cache = std::env::var("SURREAL_TRANSACTION_CACHE_SIZE")
|
||||
.unwrap_or_else(|_| (cpu_count * 16).to_string());
|
||||
std::env::set_var("SURREAL_TRANSACTION_CACHE_SIZE", transaction_cache);
|
||||
// SAFETY: single-threaded setup before SurrealDB clients are created.
|
||||
unsafe {
|
||||
std::env::set_var("SURREAL_INDEXING_BATCH_SIZE", indexing_batch_size);
|
||||
std::env::set_var(
|
||||
"SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE",
|
||||
max_order_queue,
|
||||
);
|
||||
std::env::set_var(
|
||||
"SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS",
|
||||
websocket_concurrent,
|
||||
);
|
||||
std::env::set_var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE", websocket_buffer);
|
||||
std::env::set_var("SURREAL_TRANSACTION_CACHE_SIZE", transaction_cache);
|
||||
}
|
||||
|
||||
info!(
|
||||
indexing_batch_size = %std::env::var("SURREAL_INDEXING_BATCH_SIZE").unwrap(),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use async_openai::{config::OpenAIConfig, Client};
|
||||
use async_openai::{Client, config::OpenAIConfig};
|
||||
|
||||
const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1";
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::{
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use anyhow::{Result, anyhow};
|
||||
use async_openai::Client;
|
||||
use common::{
|
||||
storage::{
|
||||
|
||||
@@ -16,12 +16,12 @@ pub(crate) async fn finalize(ctx: &mut EvaluationContext<'_>) -> anyhow::Result<
|
||||
);
|
||||
let started = Instant::now();
|
||||
|
||||
if let Some(path) = ctx.diagnostics_path.as_ref() {
|
||||
if ctx.diagnostics_enabled {
|
||||
write_chunk_diagnostics(path.as_path(), &ctx.diagnostics_output)
|
||||
.await
|
||||
.with_context(|| format!("writing chunk diagnostics to {}", path.display()))?;
|
||||
}
|
||||
if let Some(path) = ctx.diagnostics_path.as_ref()
|
||||
&& ctx.diagnostics_enabled
|
||||
{
|
||||
write_chunk_diagnostics(path.as_path(), &ctx.diagnostics_output)
|
||||
.await
|
||||
.with_context(|| format!("writing chunk diagnostics to {}", path.display()))?;
|
||||
}
|
||||
|
||||
info!(
|
||||
|
||||
@@ -40,8 +40,8 @@ pub(crate) async fn prepare_corpus(ctx: &mut EvaluationContext<'_>) -> anyhow::R
|
||||
|
||||
if !config.reseed_slice {
|
||||
let requested_cases = window.cases.len();
|
||||
if let Some(manifest) = corpus::load_cached_manifest(&base_dir)? {
|
||||
if can_reuse_namespace(
|
||||
if let Some(manifest) = corpus::load_cached_manifest(&base_dir)?
|
||||
&& can_reuse_namespace(
|
||||
ctx.db()?,
|
||||
&manifest,
|
||||
&embedding_provider,
|
||||
@@ -51,28 +51,27 @@ pub(crate) async fn prepare_corpus(ctx: &mut EvaluationContext<'_>) -> anyhow::R
|
||||
requested_cases,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
info!(
|
||||
cache = %base_dir.display(),
|
||||
namespace = ctx.namespace.as_str(),
|
||||
database = ctx.database.as_str(),
|
||||
"Namespace already seeded; reusing cached corpus manifest"
|
||||
);
|
||||
let corpus_handle = corpus::corpus_handle_from_manifest(manifest, base_dir);
|
||||
ctx.corpus_handle = Some(corpus_handle);
|
||||
ctx.expected_fingerprint = Some(expected_fingerprint);
|
||||
ctx.ingestion_duration_ms = 0;
|
||||
{
|
||||
info!(
|
||||
cache = %base_dir.display(),
|
||||
namespace = ctx.namespace.as_str(),
|
||||
database = ctx.database.as_str(),
|
||||
"Namespace already seeded; reusing cached corpus manifest"
|
||||
);
|
||||
let corpus_handle = corpus::corpus_handle_from_manifest(manifest, base_dir);
|
||||
ctx.corpus_handle = Some(corpus_handle);
|
||||
ctx.expected_fingerprint = Some(expected_fingerprint);
|
||||
ctx.ingestion_duration_ms = 0;
|
||||
|
||||
let elapsed = started.elapsed();
|
||||
ctx.record_stage_duration(stage, elapsed);
|
||||
info!(
|
||||
evaluation_stage = stage.label(),
|
||||
duration_ms = elapsed.as_millis(),
|
||||
"completed evaluation stage"
|
||||
);
|
||||
let elapsed = started.elapsed();
|
||||
ctx.record_stage_duration(stage, elapsed);
|
||||
info!(
|
||||
evaluation_stage = stage.label(),
|
||||
duration_ms = elapsed.as_millis(),
|
||||
"completed evaluation stage"
|
||||
);
|
||||
|
||||
return Ok(());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use anyhow::{Context, anyhow};
|
||||
use tracing::info;
|
||||
|
||||
use crate::{
|
||||
@@ -9,7 +9,7 @@ use crate::{
|
||||
openai,
|
||||
settings::{enforce_system_settings, load_or_init_system_settings},
|
||||
};
|
||||
use common::utils::embedding::{default_embedding_pool_size, EmbeddingProvider};
|
||||
use common::utils::embedding::{EmbeddingProvider, default_embedding_pool_size};
|
||||
|
||||
use super::super::context::{EvalStage, EvaluationContext};
|
||||
|
||||
@@ -65,16 +65,16 @@ pub(crate) async fn prepare_db(ctx: &mut EvaluationContext<'_>) -> anyhow::Resul
|
||||
let (mut settings, settings_missing) =
|
||||
load_or_init_system_settings(&db, provider_dimension).await?;
|
||||
|
||||
if config.embedding_backend == EmbeddingBackend::FastEmbed {
|
||||
if let Some(model_code) = embedding_provider.model_code() {
|
||||
let sanitized = sanitize_model_code(&model_code);
|
||||
let path = config.cache_dir.join(format!("{sanitized}.json"));
|
||||
if config.force_convert && path.exists() {
|
||||
tokio::fs::remove_file(&path)
|
||||
.await
|
||||
.with_context(|| format!("removing stale cache {}", path.display()))
|
||||
.ok();
|
||||
}
|
||||
if config.embedding_backend == EmbeddingBackend::FastEmbed
|
||||
&& let Some(model_code) = embedding_provider.model_code()
|
||||
{
|
||||
let sanitized = sanitize_model_code(&model_code);
|
||||
let path = config.cache_dir.join(format!("{sanitized}.json"));
|
||||
if config.force_convert && path.exists() {
|
||||
tokio::fs::remove_file(&path)
|
||||
.await
|
||||
.with_context(|| format!("removing stale cache {}", path.display()))
|
||||
.ok();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::time::Instant;
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use anyhow::{Context, anyhow};
|
||||
use common::storage::types::system_settings::SystemSettings;
|
||||
use tracing::{info, warn};
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::{collections::HashSet, sync::Arc, time::Instant};
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use anyhow::{Context, anyhow};
|
||||
use common::storage::types::StoredObject;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use tracing::{debug, info};
|
||||
@@ -9,8 +9,8 @@ use crate::{
|
||||
cases::SeededCase,
|
||||
context_stats,
|
||||
types::{
|
||||
adapt_retrieval_output, build_case_diagnostics, text_contains_answer, CaseDiagnostics,
|
||||
CaseSummary, RetrievedSummary,
|
||||
CaseDiagnostics, CaseSummary, RetrievedSummary, adapt_retrieval_output,
|
||||
build_case_diagnostics, text_contains_answer,
|
||||
},
|
||||
};
|
||||
use retrieval_pipeline::{
|
||||
@@ -391,9 +391,5 @@ fn calculate_ndcg(retrieved: &[RetrievedSummary], k: usize) -> f64 {
|
||||
idcg += rel / (f64::from(i) + 2.0).log2();
|
||||
}
|
||||
|
||||
if idcg == 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
dcg / idcg
|
||||
}
|
||||
if idcg == 0.0 { 0.0 } else { dcg / idcg }
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ use chrono::Utc;
|
||||
use tracing::info;
|
||||
|
||||
use crate::types::{
|
||||
build_stage_latency_breakdown, compute_latency_stats, EvaluationSummary, PerformanceTimings,
|
||||
RetrievedContextStats,
|
||||
EvaluationSummary, PerformanceTimings, RetrievedContextStats, build_stage_latency_breakdown,
|
||||
compute_latency_stats,
|
||||
};
|
||||
|
||||
use super::super::context::{EvalStage, EvaluationContext};
|
||||
|
||||
@@ -8,8 +8,8 @@ use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::types::{
|
||||
format_timestamp, CaseSummary, EvaluationStageTimings, EvaluationSummary, LatencyStats,
|
||||
RetrievalContextStats, StageLatencyBreakdown,
|
||||
CaseSummary, EvaluationStageTimings, EvaluationSummary, LatencyStats, RetrievalContextStats,
|
||||
StageLatencyBreakdown, format_timestamp,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -804,11 +804,7 @@ fn prettify_stage(label: &str) -> String {
|
||||
}
|
||||
|
||||
fn bool_badge(value: bool) -> &'static str {
|
||||
if value {
|
||||
"✅"
|
||||
} else {
|
||||
"⚪"
|
||||
}
|
||||
if value { "✅" } else { "⚪" }
|
||||
}
|
||||
|
||||
fn render_retrieved(entries: &[RetrievedSnippet]) -> String {
|
||||
|
||||
@@ -24,15 +24,15 @@ pub(crate) async fn enforce_system_settings(
|
||||
updated_settings.embedding_dimensions = provider_dimension as u32;
|
||||
needs_settings_update = true;
|
||||
}
|
||||
if let Some(query_override) = config.query_model.as_deref() {
|
||||
if settings.query_model != query_override {
|
||||
info!(
|
||||
model = query_override,
|
||||
"Overriding system query model for this run"
|
||||
);
|
||||
updated_settings.query_model = query_override.to_string();
|
||||
needs_settings_update = true;
|
||||
}
|
||||
if let Some(query_override) = config.query_model.as_deref()
|
||||
&& settings.query_model != query_override
|
||||
{
|
||||
info!(
|
||||
model = query_override,
|
||||
"Overriding system query model for this run"
|
||||
);
|
||||
updated_settings.query_model = query_override.to_string();
|
||||
needs_settings_update = true;
|
||||
}
|
||||
if needs_settings_update {
|
||||
settings = SystemSettings::update(db, updated_settings)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
|
||||
use anyhow::{Result, anyhow};
|
||||
use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::datasets::{ConvertedDataset, BEIR_DATASETS};
|
||||
use crate::datasets::{BEIR_DATASETS, ConvertedDataset};
|
||||
|
||||
use super::build::{mix_seed, BuildParams};
|
||||
use super::build::{BuildParams, mix_seed};
|
||||
|
||||
#[allow(clippy::too_many_lines, clippy::arithmetic_side_effects)]
|
||||
pub(super) fn ordered_question_refs_beir(
|
||||
@@ -164,10 +164,10 @@ pub(super) fn ordered_question_refs_beir(
|
||||
|
||||
pub(super) fn question_prefix(question_id: &str) -> Option<&'static str> {
|
||||
for prefix in BEIR_DATASETS.iter().map(|kind| kind.source_prefix()) {
|
||||
if let Some(rest) = question_id.strip_prefix(prefix) {
|
||||
if rest.starts_with('-') {
|
||||
return Some(prefix);
|
||||
}
|
||||
if let Some(rest) = question_id.strip_prefix(prefix)
|
||||
&& rest.starts_with('-')
|
||||
{
|
||||
return Some(prefix);
|
||||
}
|
||||
}
|
||||
None
|
||||
|
||||
@@ -5,9 +5,9 @@ use std::{
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use chrono::{DateTime, Utc};
|
||||
use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
|
||||
use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::{info, warn};
|
||||
@@ -20,7 +20,7 @@ use crate::{
|
||||
mod beir;
|
||||
mod build;
|
||||
|
||||
use build::{mix_seed, BuildParams};
|
||||
use build::{BuildParams, mix_seed};
|
||||
|
||||
const SLICE_VERSION: u32 = 2;
|
||||
pub const DEFAULT_NEGATIVE_MULTIPLIER: f32 = 4.0;
|
||||
@@ -1116,11 +1116,13 @@ mod tests {
|
||||
assert_eq!(window.cases.len(), 1);
|
||||
let positive_ids: Vec<&str> = window.positive_ids().collect();
|
||||
assert_eq!(positive_ids.len(), 1);
|
||||
assert!(resolved
|
||||
.manifest
|
||||
.paragraphs
|
||||
.iter()
|
||||
.any(|entry| entry.id == positive_ids[0]));
|
||||
assert!(
|
||||
resolved
|
||||
.manifest
|
||||
.paragraphs
|
||||
.iter()
|
||||
.any(|entry| entry.id == positive_ids[0])
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user