release: 1.0.5

This commit is contained in:
Per Stark
2026-06-24 22:02:31 +02:00
parent ba3fd6ed46
commit d273390de8
118 changed files with 989 additions and 690 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "evaluations"
version = "0.1.0"
edition = "2021"
edition = "2024"
[lints]
workspace = true
+45 -46
View File
@@ -3,7 +3,7 @@ use std::{
path::{Path, PathBuf},
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use clap::{Args, Parser, ValueEnum};
use crate::datasets::DatasetKind;
@@ -394,26 +394,26 @@ impl Config {
));
}
if let Some(k) = self.retrieval.chunk_rrf_k {
if k <= 0.0 || !k.is_finite() {
return Err(anyhow!(
"--chunk-rrf-k must be a positive, finite number (got {k})"
));
}
if let Some(k) = self.retrieval.chunk_rrf_k
&& (k <= 0.0 || !k.is_finite())
{
return Err(anyhow!(
"--chunk-rrf-k must be a positive, finite number (got {k})"
));
}
if let Some(weight) = self.retrieval.chunk_rrf_vector_weight {
if weight < 0.0 || !weight.is_finite() {
return Err(anyhow!(
"--chunk-rrf-vector-weight must be a non-negative, finite number (got {weight})"
));
}
if let Some(weight) = self.retrieval.chunk_rrf_vector_weight
&& (weight < 0.0 || !weight.is_finite())
{
return Err(anyhow!(
"--chunk-rrf-vector-weight must be a non-negative, finite number (got {weight})"
));
}
if let Some(weight) = self.retrieval.chunk_rrf_fts_weight {
if weight < 0.0 || !weight.is_finite() {
return Err(anyhow!(
"--chunk-rrf-fts-weight must be a non-negative, finite number (got {weight})"
));
}
if let Some(weight) = self.retrieval.chunk_rrf_fts_weight
&& (weight < 0.0 || !weight.is_finite())
{
return Err(anyhow!(
"--chunk-rrf-fts-weight must be a non-negative, finite number (got {weight})"
));
}
if self.concurrency == 0 {
@@ -426,16 +426,16 @@ impl Config {
));
}
if let Some(query_model) = &self.query_model {
if query_model.trim().is_empty() {
return Err(anyhow!("--query-model requires a non-empty model name"));
}
if let Some(query_model) = &self.query_model
&& query_model.trim().is_empty()
{
return Err(anyhow!("--query-model requires a non-empty model name"));
}
if let Some(grow) = self.slice_grow {
if grow == 0 {
return Err(anyhow!("--slice-grow must be greater than zero"));
}
if let Some(grow) = self.slice_grow
&& grow == 0
{
return Err(anyhow!("--slice-grow must be greater than zero"));
}
if self.negative_multiplier <= 0.0 || !self.negative_multiplier.is_finite() {
@@ -465,12 +465,11 @@ impl Config {
}
// Handle perf log dir env var fallback
if self.perf_log_dir.is_none() {
if let Ok(dir) = env::var("EVAL_PERF_LOG_DIR") {
if !dir.trim().is_empty() {
self.perf_log_dir = Some(PathBuf::from(dir));
}
}
if self.perf_log_dir.is_none()
&& let Ok(dir) = env::var("EVAL_PERF_LOG_DIR")
&& !dir.trim().is_empty()
{
self.perf_log_dir = Some(PathBuf::from(dir));
}
Ok(())
@@ -480,10 +479,10 @@ impl Config {
let catalog = crate::datasets::catalog()?;
let entry = catalog.dataset(self.dataset.id())?;
if self.slice.is_none() {
if let Some(default_slice) = entry.slices.first() {
self.slice = Some(default_slice.id.clone());
}
if self.slice.is_none()
&& let Some(default_slice) = entry.slices.first()
{
self.slice = Some(default_slice.id.clone());
}
let Some(slice_id) = self.slice.as_deref() else {
@@ -498,11 +497,11 @@ impl Config {
return Ok(());
}
if let Some(limit) = slice.limit {
if self.limit_arg == 200 {
self.limit_arg = limit;
self.limit = Some(limit);
}
if let Some(limit) = slice.limit
&& self.limit_arg == 200
{
self.limit_arg = limit;
self.limit = Some(limit);
}
if self.corpus_limit.is_none() {
self.corpus_limit = slice.corpus_limit;
@@ -514,10 +513,10 @@ impl Config {
self.llm_mode = include_unanswerable;
self.retrieval.require_verified_chunks = !include_unanswerable;
}
if let Some(multiplier) = slice.negative_multiplier {
if negative_multiplier_is_default(self.negative_multiplier) {
self.negative_multiplier = multiplier;
}
if let Some(multiplier) = slice.negative_multiplier
&& negative_multiplier_is_default(self.negative_multiplier)
{
self.negative_multiplier = multiplier;
}
Ok(())
}
+2 -2
View File
@@ -9,8 +9,8 @@ use crate::{
args::Config,
corpus::{self, CorpusCacheConfig},
datasets::{
beir_subset_store_summary, beir_subset_stores_ready, content_checksum_for_layout,
detect_layout, mix_content_checksum, store_dir_for, ConvertedLayout, DatasetKind,
ConvertedLayout, DatasetKind, beir_subset_store_summary, beir_subset_stores_ready,
content_checksum_for_layout, detect_layout, mix_content_checksum, store_dir_for,
},
db::{connect_eval_db, default_database, default_namespace, namespace_has_corpus},
slice::{self, ledger_target},
+3 -2
View File
@@ -8,8 +8,9 @@ pub use orchestrator::{
load_cached_manifest, persist_corpus_manifest,
};
pub use store::{
seed_manifest_into_db, window_manifest, CorpusHandle, CorpusManifest, CorpusMetadata,
CorpusQuestion, NamespaceSeedRecord, ParagraphShard, ParagraphShardStore, MANIFEST_VERSION,
CorpusHandle, CorpusManifest, CorpusMetadata, CorpusQuestion, MANIFEST_VERSION,
NamespaceSeedRecord, ParagraphShard, ParagraphShardStore, seed_manifest_into_db,
window_manifest,
};
pub fn make_ingestion_config(config: &crate::args::Config) -> ingestion_pipeline::IngestionConfig {
+3 -3
View File
@@ -6,14 +6,14 @@ use std::{
sync::Arc,
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use async_openai::Client;
use chrono::Utc;
use common::{
storage::{
db::SurrealDbClient,
store::{DynStorage, StorageManager},
types::{ingestion_payload::IngestionPayload, ingestion_task::IngestionTask, StoredObject},
types::{StoredObject, ingestion_payload::IngestionPayload, ingestion_task::IngestionTask},
},
utils::config::{AppConfig, StorageKind},
};
@@ -31,7 +31,7 @@ use crate::{
use crate::corpus::{
CorpusCacheConfig, CorpusHandle, CorpusManifest, CorpusMetadata, CorpusQuestion,
ParagraphShard, ParagraphShardStore, MANIFEST_VERSION,
MANIFEST_VERSION, ParagraphShard, ParagraphShardStore,
};
const INGESTION_SPEC_VERSION: u32 = 2;
+6 -5
View File
@@ -5,16 +5,17 @@ use std::{
path::PathBuf,
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
use common::storage::{
db::SurrealDbClient,
types::{
knowledge_entity::KnowledgeEntity, knowledge_relationship::KnowledgeRelationship,
text_chunk::TextChunk, text_content::TextContent, StoredObject,
StoredObject, knowledge_entity::KnowledgeEntity,
knowledge_relationship::KnowledgeRelationship, text_chunk::TextChunk,
text_content::TextContent,
},
};
use ingestion_pipeline::{persist_artifacts, IngestionTuning, PipelineArtifacts};
use ingestion_pipeline::{IngestionTuning, PipelineArtifacts, persist_artifacts};
use serde::Deserialize;
use tracing::{debug, warn};
@@ -304,7 +305,7 @@ impl ParagraphShardStore {
Ok(file) => file,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(err) => {
return Err(err).with_context(|| format!("opening shard {}", path.display()))
return Err(err).with_context(|| format!("opening shard {}", path.display()));
}
};
let reader = BufReader::new(file);
+5 -5
View File
@@ -5,7 +5,7 @@ use std::{
path::{Path, PathBuf},
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use serde::Deserialize;
use tracing::warn;
@@ -138,10 +138,10 @@ pub fn convert_beir_documents(
continue;
};
if let Some(filter) = doc_ids {
if !filter.contains(&best.doc_id) {
continue;
}
if let Some(filter) = doc_ids
&& !filter.contains(&best.doc_id)
{
continue;
}
let Some(&paragraph_slot) = paragraph_index.get(&best.doc_id) else {
+2 -3
View File
@@ -1,17 +1,16 @@
use std::collections::{HashMap, HashSet};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use sha2::{Digest, Sha256};
use tracing::info;
use super::{
beir,
BEIR_DATASETS, ConvertedDataset, DatasetKind, DatasetMetadata, beir,
checksum::hash_file,
store::{
self, build_dataset_from_catalog, paragraph_path, read_meta, store_dir_for,
upsert_sharded_paragraphs, write_sharded,
},
ConvertedDataset, DatasetKind, DatasetMetadata, BEIR_DATASETS,
};
use crate::{args::Config, slice};
+15 -17
View File
@@ -112,10 +112,10 @@ pub fn write_sidecar(content_path: &Path, sha256: &str) -> Result<()> {
#[cfg(test)]
pub fn content_checksum(content_path: &Path) -> Result<String> {
let sidecar_path = ChecksumSidecar::sidecar_path(content_path);
if let Some(sidecar) = read_sidecar(&sidecar_path)? {
if sidecar.is_valid_for(content_path) {
return Ok(sidecar.sha256);
}
if let Some(sidecar) = read_sidecar(&sidecar_path)?
&& sidecar.is_valid_for(content_path)
{
return Ok(sidecar.sha256);
}
let sha256 = hash_file(content_path)?;
write_sidecar(content_path, &sha256)?;
@@ -125,19 +125,17 @@ pub fn content_checksum(content_path: &Path) -> Result<String> {
pub fn store_aggregate_checksum(store_dir: &Path) -> Result<String> {
let marker = store_dir.join("checksum.sha256");
let meta = store_dir.join("meta.json");
if marker.is_file() && meta.is_file() {
if let (Ok(marker_meta), Ok(meta_meta)) = (marker.metadata(), meta.metadata()) {
if marker_meta
.modified()
.ok()
.zip(meta_meta.modified().ok())
.is_some_and(|(marker_modified, meta_modified)| marker_modified >= meta_modified)
{
if let Some(sidecar) = read_sidecar(&marker)? {
return Ok(sidecar.sha256);
}
}
}
if marker.is_file()
&& meta.is_file()
&& let (Ok(marker_meta), Ok(meta_meta)) = (marker.metadata(), meta.metadata())
&& marker_meta
.modified()
.ok()
.zip(meta_meta.modified().ok())
.is_some_and(|(marker_modified, meta_modified)| marker_modified >= meta_modified)
&& let Some(sidecar) = read_sidecar(&marker)?
{
return Ok(sidecar.sha256);
}
let mut entries = Vec::new();
+16 -19
View File
@@ -4,12 +4,11 @@ use anyhow::{Context, Result};
use tracing::info;
use super::{
catalog,
ConvertedDataset, DatasetKind, catalog,
store::{
self, build_dataset_from_catalog, detect_layout, read_meta, store_dir_for, write_sharded,
ConvertedLayout,
self, ConvertedLayout, build_dataset_from_catalog, detect_layout, read_meta, store_dir_for,
write_sharded,
},
ConvertedDataset, DatasetKind,
};
use crate::{
args::Config,
@@ -69,21 +68,19 @@ fn load_from_store(
let meta = read_meta(store_dir)?;
validate_metadata_fields(&meta.metadata, dataset_kind, config)?;
if allow_partial {
if let Some(paragraph_ids) = slice_paragraph_ids_for_fast_path(config)? {
let unique: HashSet<String> = paragraph_ids.into_iter().collect();
info!(
paragraphs = unique.len(),
store = %store_dir.display(),
"Loading slice-addressed paragraphs from sharded converted store"
);
let dataset = build_dataset_from_catalog(store_dir, &unique)?;
return Ok(LoadedDataset {
dataset,
content_checksum: checksum,
partial: true,
});
}
if allow_partial && let Some(paragraph_ids) = slice_paragraph_ids_for_fast_path(config)? {
let unique: HashSet<String> = paragraph_ids.into_iter().collect();
info!(
paragraphs = unique.len(),
store = %store_dir.display(),
"Loading slice-addressed paragraphs from sharded converted store"
);
let dataset = build_dataset_from_catalog(store_dir, &unique)?;
return Ok(LoadedDataset {
dataset,
content_checksum: checksum,
partial: true,
});
}
info!(
+5 -3
View File
@@ -13,7 +13,7 @@ use std::{
str::FromStr,
};
use anyhow::{anyhow, bail, Context, Result};
use anyhow::{Context, Result, anyhow, bail};
use chrono::{DateTime, TimeZone, Utc};
use clap::ValueEnum;
use once_cell::sync::OnceCell;
@@ -226,7 +226,7 @@ pub use beir_mix::{beir_subset_store_summary, beir_subset_stores_ready, mix_cont
pub use checksum::store_aggregate_checksum;
pub use loader::{prebuild_catalog_slices, prepare_dataset};
pub use store::{
content_checksum_for_layout, detect_layout, store_dir_for, write_sharded, ConvertedLayout,
ConvertedLayout, content_checksum_for_layout, detect_layout, store_dir_for, write_sharded,
};
pub fn catalog() -> Result<&'static DatasetCatalog> {
@@ -383,7 +383,9 @@ impl FromStr for DatasetKind {
"scifact" => Ok(Self::Scifact),
"nq-beir" | "natural-questions-beir" => Ok(Self::NqBeir),
other => {
anyhow::bail!("unknown dataset '{other}'. Expected one of: squad, natural-questions, beir, fever, fiqa, hotpotqa, nfcorpus, quora, trec-covid, scifact, nq-beir.")
anyhow::bail!(
"unknown dataset '{other}'. Expected one of: squad, natural-questions, beir, fever, fiqa, hotpotqa, nfcorpus, quora, trec-covid, scifact, nq-beir."
)
}
}
}
+3 -3
View File
@@ -5,14 +5,14 @@ use std::{
path::{Path, PathBuf},
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use tracing::info;
use super::{
checksum::store_aggregate_checksum, ConvertedDataset, ConvertedParagraph, ConvertedQuestion,
DatasetMetadata,
ConvertedDataset, ConvertedParagraph, ConvertedQuestion, DatasetMetadata,
checksum::store_aggregate_checksum,
};
use crate::slice;
+2 -2
View File
@@ -1,10 +1,10 @@
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use chrono::Utc;
use common::{
storage::{
db::SurrealDbClient,
types::user::{Theme, User},
types::StoredObject,
types::user::{Theme, User},
},
utils::embedding::EmbeddingProvider,
};
+1 -1
View File
@@ -1,6 +1,6 @@
use std::{collections::HashMap, fs, path::Path};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use common::storage::{db::SurrealDbClient, types::text_chunk::TextChunk};
use crate::{args::Config, corpus, db::connect_eval_db};
+15 -16
View File
@@ -17,36 +17,35 @@ mod types;
use anyhow::Context;
use tokio::runtime::Builder;
use tracing::info;
use tracing_subscriber::{fmt, EnvFilter};
use tracing_subscriber::{EnvFilter, fmt};
/// Configure `SurrealDB` environment variables for optimal performance
#[allow(clippy::arithmetic_side_effects, clippy::unwrap_used)]
fn configure_surrealdb_performance(cpu_count: usize) {
let indexing_batch_size = std::env::var("SURREAL_INDEXING_BATCH_SIZE")
.unwrap_or_else(|_| (cpu_count * 2).to_string());
std::env::set_var("SURREAL_INDEXING_BATCH_SIZE", indexing_batch_size);
let max_order_queue = std::env::var("SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE")
.unwrap_or_else(|_| (cpu_count * 4).to_string());
std::env::set_var(
"SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE",
max_order_queue,
);
let websocket_concurrent = std::env::var("SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS")
.unwrap_or_else(|_| cpu_count.to_string());
std::env::set_var(
"SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS",
websocket_concurrent,
);
let websocket_buffer = std::env::var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE")
.unwrap_or_else(|_| (cpu_count * 8).to_string());
std::env::set_var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE", websocket_buffer);
let transaction_cache = std::env::var("SURREAL_TRANSACTION_CACHE_SIZE")
.unwrap_or_else(|_| (cpu_count * 16).to_string());
std::env::set_var("SURREAL_TRANSACTION_CACHE_SIZE", transaction_cache);
// SAFETY: single-threaded setup before SurrealDB clients are created.
unsafe {
std::env::set_var("SURREAL_INDEXING_BATCH_SIZE", indexing_batch_size);
std::env::set_var(
"SURREAL_MAX_ORDER_LIMIT_PRIORITY_QUEUE_SIZE",
max_order_queue,
);
std::env::set_var(
"SURREAL_WEBSOCKET_MAX_CONCURRENT_REQUESTS",
websocket_concurrent,
);
std::env::set_var("SURREAL_WEBSOCKET_RESPONSE_BUFFER_SIZE", websocket_buffer);
std::env::set_var("SURREAL_TRANSACTION_CACHE_SIZE", transaction_cache);
}
info!(
indexing_batch_size = %std::env::var("SURREAL_INDEXING_BATCH_SIZE").unwrap(),
+1 -1
View File
@@ -1,7 +1,7 @@
use std::sync::Arc;
use anyhow::{Context, Result};
use async_openai::{config::OpenAIConfig, Client};
use async_openai::{Client, config::OpenAIConfig};
const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1";
+1 -1
View File
@@ -4,7 +4,7 @@ use std::{
time::{Duration, Instant},
};
use anyhow::{anyhow, Result};
use anyhow::{Result, anyhow};
use async_openai::Client;
use common::{
storage::{
+6 -6
View File
@@ -16,12 +16,12 @@ pub(crate) async fn finalize(ctx: &mut EvaluationContext<'_>) -> anyhow::Result<
);
let started = Instant::now();
if let Some(path) = ctx.diagnostics_path.as_ref() {
if ctx.diagnostics_enabled {
write_chunk_diagnostics(path.as_path(), &ctx.diagnostics_output)
.await
.with_context(|| format!("writing chunk diagnostics to {}", path.display()))?;
}
if let Some(path) = ctx.diagnostics_path.as_ref()
&& ctx.diagnostics_enabled
{
write_chunk_diagnostics(path.as_path(), &ctx.diagnostics_output)
.await
.with_context(|| format!("writing chunk diagnostics to {}", path.display()))?;
}
info!(
@@ -40,8 +40,8 @@ pub(crate) async fn prepare_corpus(ctx: &mut EvaluationContext<'_>) -> anyhow::R
if !config.reseed_slice {
let requested_cases = window.cases.len();
if let Some(manifest) = corpus::load_cached_manifest(&base_dir)? {
if can_reuse_namespace(
if let Some(manifest) = corpus::load_cached_manifest(&base_dir)?
&& can_reuse_namespace(
ctx.db()?,
&manifest,
&embedding_provider,
@@ -51,28 +51,27 @@ pub(crate) async fn prepare_corpus(ctx: &mut EvaluationContext<'_>) -> anyhow::R
requested_cases,
)
.await?
{
info!(
cache = %base_dir.display(),
namespace = ctx.namespace.as_str(),
database = ctx.database.as_str(),
"Namespace already seeded; reusing cached corpus manifest"
);
let corpus_handle = corpus::corpus_handle_from_manifest(manifest, base_dir);
ctx.corpus_handle = Some(corpus_handle);
ctx.expected_fingerprint = Some(expected_fingerprint);
ctx.ingestion_duration_ms = 0;
{
info!(
cache = %base_dir.display(),
namespace = ctx.namespace.as_str(),
database = ctx.database.as_str(),
"Namespace already seeded; reusing cached corpus manifest"
);
let corpus_handle = corpus::corpus_handle_from_manifest(manifest, base_dir);
ctx.corpus_handle = Some(corpus_handle);
ctx.expected_fingerprint = Some(expected_fingerprint);
ctx.ingestion_duration_ms = 0;
let elapsed = started.elapsed();
ctx.record_stage_duration(stage, elapsed);
info!(
evaluation_stage = stage.label(),
duration_ms = elapsed.as_millis(),
"completed evaluation stage"
);
let elapsed = started.elapsed();
ctx.record_stage_duration(stage, elapsed);
info!(
evaluation_stage = stage.label(),
duration_ms = elapsed.as_millis(),
"completed evaluation stage"
);
return Ok(());
}
return Ok(());
}
}
+12 -12
View File
@@ -1,6 +1,6 @@
use std::time::Instant;
use anyhow::{anyhow, Context};
use anyhow::{Context, anyhow};
use tracing::info;
use crate::{
@@ -9,7 +9,7 @@ use crate::{
openai,
settings::{enforce_system_settings, load_or_init_system_settings},
};
use common::utils::embedding::{default_embedding_pool_size, EmbeddingProvider};
use common::utils::embedding::{EmbeddingProvider, default_embedding_pool_size};
use super::super::context::{EvalStage, EvaluationContext};
@@ -65,16 +65,16 @@ pub(crate) async fn prepare_db(ctx: &mut EvaluationContext<'_>) -> anyhow::Resul
let (mut settings, settings_missing) =
load_or_init_system_settings(&db, provider_dimension).await?;
if config.embedding_backend == EmbeddingBackend::FastEmbed {
if let Some(model_code) = embedding_provider.model_code() {
let sanitized = sanitize_model_code(&model_code);
let path = config.cache_dir.join(format!("{sanitized}.json"));
if config.force_convert && path.exists() {
tokio::fs::remove_file(&path)
.await
.with_context(|| format!("removing stale cache {}", path.display()))
.ok();
}
if config.embedding_backend == EmbeddingBackend::FastEmbed
&& let Some(model_code) = embedding_provider.model_code()
{
let sanitized = sanitize_model_code(&model_code);
let path = config.cache_dir.join(format!("{sanitized}.json"));
if config.force_convert && path.exists() {
tokio::fs::remove_file(&path)
.await
.with_context(|| format!("removing stale cache {}", path.display()))
.ok();
}
}
@@ -1,6 +1,6 @@
use std::time::Instant;
use anyhow::{anyhow, Context};
use anyhow::{Context, anyhow};
use common::storage::types::system_settings::SystemSettings;
use tracing::{info, warn};
@@ -1,6 +1,6 @@
use std::{collections::HashSet, sync::Arc, time::Instant};
use anyhow::{anyhow, Context};
use anyhow::{Context, anyhow};
use common::storage::types::StoredObject;
use futures::stream::{self, StreamExt};
use tracing::{debug, info};
@@ -9,8 +9,8 @@ use crate::{
cases::SeededCase,
context_stats,
types::{
adapt_retrieval_output, build_case_diagnostics, text_contains_answer, CaseDiagnostics,
CaseSummary, RetrievedSummary,
CaseDiagnostics, CaseSummary, RetrievedSummary, adapt_retrieval_output,
build_case_diagnostics, text_contains_answer,
},
};
use retrieval_pipeline::{
@@ -391,9 +391,5 @@ fn calculate_ndcg(retrieved: &[RetrievedSummary], k: usize) -> f64 {
idcg += rel / (f64::from(i) + 2.0).log2();
}
if idcg == 0.0 {
0.0
} else {
dcg / idcg
}
if idcg == 0.0 { 0.0 } else { dcg / idcg }
}
+2 -2
View File
@@ -4,8 +4,8 @@ use chrono::Utc;
use tracing::info;
use crate::types::{
build_stage_latency_breakdown, compute_latency_stats, EvaluationSummary, PerformanceTimings,
RetrievedContextStats,
EvaluationSummary, PerformanceTimings, RetrievedContextStats, build_stage_latency_breakdown,
compute_latency_stats,
};
use super::super::context::{EvalStage, EvaluationContext};
+3 -7
View File
@@ -8,8 +8,8 @@ use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::types::{
format_timestamp, CaseSummary, EvaluationStageTimings, EvaluationSummary, LatencyStats,
RetrievalContextStats, StageLatencyBreakdown,
CaseSummary, EvaluationStageTimings, EvaluationSummary, LatencyStats, RetrievalContextStats,
StageLatencyBreakdown, format_timestamp,
};
#[derive(Debug)]
@@ -804,11 +804,7 @@ fn prettify_stage(label: &str) -> String {
}
fn bool_badge(value: bool) -> &'static str {
if value {
""
} else {
""
}
if value { "" } else { "" }
}
fn render_retrieved(entries: &[RetrievedSnippet]) -> String {
+9 -9
View File
@@ -24,15 +24,15 @@ pub(crate) async fn enforce_system_settings(
updated_settings.embedding_dimensions = provider_dimension as u32;
needs_settings_update = true;
}
if let Some(query_override) = config.query_model.as_deref() {
if settings.query_model != query_override {
info!(
model = query_override,
"Overriding system query model for this run"
);
updated_settings.query_model = query_override.to_string();
needs_settings_update = true;
}
if let Some(query_override) = config.query_model.as_deref()
&& settings.query_model != query_override
{
info!(
model = query_override,
"Overriding system query model for this run"
);
updated_settings.query_model = query_override.to_string();
needs_settings_update = true;
}
if needs_settings_update {
settings = SystemSettings::update(db, updated_settings)
+8 -8
View File
@@ -1,12 +1,12 @@
use std::collections::{HashMap, VecDeque};
use anyhow::{anyhow, Result};
use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
use anyhow::{Result, anyhow};
use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
use tracing::warn;
use crate::datasets::{ConvertedDataset, BEIR_DATASETS};
use crate::datasets::{BEIR_DATASETS, ConvertedDataset};
use super::build::{mix_seed, BuildParams};
use super::build::{BuildParams, mix_seed};
#[allow(clippy::too_many_lines, clippy::arithmetic_side_effects)]
pub(super) fn ordered_question_refs_beir(
@@ -164,10 +164,10 @@ pub(super) fn ordered_question_refs_beir(
pub(super) fn question_prefix(question_id: &str) -> Option<&'static str> {
for prefix in BEIR_DATASETS.iter().map(|kind| kind.source_prefix()) {
if let Some(rest) = question_id.strip_prefix(prefix) {
if rest.starts_with('-') {
return Some(prefix);
}
if let Some(rest) = question_id.strip_prefix(prefix)
&& rest.starts_with('-')
{
return Some(prefix);
}
}
None
+10 -8
View File
@@ -5,9 +5,9 @@ use std::{
path::{Path, PathBuf},
};
use anyhow::{anyhow, Context, Result};
use anyhow::{Context, Result, anyhow};
use chrono::{DateTime, Utc};
use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tracing::{info, warn};
@@ -20,7 +20,7 @@ use crate::{
mod beir;
mod build;
use build::{mix_seed, BuildParams};
use build::{BuildParams, mix_seed};
const SLICE_VERSION: u32 = 2;
pub const DEFAULT_NEGATIVE_MULTIPLIER: f32 = 4.0;
@@ -1116,11 +1116,13 @@ mod tests {
assert_eq!(window.cases.len(), 1);
let positive_ids: Vec<&str> = window.positive_ids().collect();
assert_eq!(positive_ids.len(), 1);
assert!(resolved
.manifest
.paragraphs
.iter()
.any(|entry| entry.id == positive_ids[0]));
assert!(
resolved
.manifest
.paragraphs
.iter()
.any(|entry| entry.id == positive_ids[0])
);
Ok(())
}