POC: Explore request assertions and CLI testing

This commit is contained in:
Gregory Schier committed 2026-09-25 08:32:24 -07:00
1 parent 263a215f7a
commit c71256d800
65 files changed
+4187 -1983

No files matched your search

+1
View File
@@ -9,6 +9,7 @@ name = "yaak"
path = "src/main.rs"
[dependencies]
yaak-assertions = { workspace = true }
arboard = "3"
base64 = "0.22"
clap = { version = "4", features = ["derive"] }
+21
View File
@@ -71,6 +71,9 @@ pub enum Commands {
/// Send a request, folder, or workspace by ID
Send(SendArgs),
/// Run HTTP assertions (exit 0: passed, 1: failed/error, 2: invalid configuration)
Test(TestArgs),
/// Cookie jar commands
CookieJar(CookieJarArgs),
@@ -186,6 +189,24 @@ pub enum AgentCommands {
},
}
#[derive(Args)]
pub struct TestArgs {
/// Request, folder, or workspace ID; sends HTTP requests sequentially in sidebar order
pub id: String,
/// Stop after the first failed check or request error
#[arg(long)]
pub fail_fast: bool,
/// Write a JSON report without URLs, headers, bodies, or assertion values
#[arg(long, value_name = "FILE")]
pub json: Option<PathBuf>,
/// Write a JUnit XML report without request or assertion values
#[arg(long, value_name = "FILE")]
pub junit: Option<PathBuf>,
}
#[derive(Args)]
pub struct SendArgs {
/// Request, folder, or workspace ID
+2
View File
@@ -10,3 +10,5 @@ pub mod response;
pub mod send;
pub mod template_function;
pub mod workspace;
pub mod test;
+442
View File
@@ -0,0 +1,442 @@
use crate::cli::TestArgs;
use crate::commands::request::resolve_cookie_jar_id;
use crate::context::CliContext;
use serde::Serialize;
use std::collections::HashSet;
use yaak::send::{SendHttpRequestWithPluginsParams, send_http_request_with_plugins};
use yaak_assertions::{AssertionReport, HttpAssertions};
use yaak_models::models::{Folder, HttpRequest};
use yaak_models::queries::any_request::AnyRequest;
use yaak_models::util::UpdateSource;
use yaak_plugins::events::PluginContext;
// Reports intentionally contain no names, URLs, selectors, expected/actual values, or transport
// error strings (which can embed URLs/tokens). The stored response has details for local inspection.
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct CheckResult {
assertion_id: String,
outcome: String,
reason: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct RequestResult {
request_id: String,
outcome: String,
reason: String,
checks: Vec<CheckResult>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct RunReport {
version: u32,
exit_code: i32,
requests: Vec<RequestResult>,
}
pub async fn run(
ctx: &CliContext,
args: TestArgs,
environment: Option<&str>,
cookie_jar: Option<&str>,
) -> i32 {
match run_inner(ctx, &args, environment, cookie_jar).await {
Ok(report) => {
if let Err(e) = write_reports(&args, &report) {
eprintln!("Could not write test report: {e}");
return 2;
}
for request in &report.requests {
println!(
"{} {}: {}",
request.outcome.to_uppercase(),
request.request_id,
request.reason
);
for check in &request.checks {
println!(
" {} {}: {}",
check.outcome.to_uppercase(),
check.assertion_id,
check.reason
);
}
}
let count = |outcome| report.requests.iter().filter(|r| r.outcome == outcome).count();
println!(
"Test summary: {} passed, {} failed, {} errors, {} unchecked, {} skipped",
count("passed"),
count("failed"),
count("error"),
count("unchecked"),
count("skipped")
);
report.exit_code
}
Err(error) => {
eprintln!("Invalid test configuration: {error}");
2
}
}
}
async fn run_inner(
ctx: &CliContext,
args: &TestArgs,
environment: Option<&str>,
cookie_jar: Option<&str>,
) -> Result<RunReport, String> {
let (workspace_id, requests) = select_requests(ctx, &args.id)?;
let jar = resolve_cookie_jar_id(ctx, &workspace_id, cookie_jar)?;
if let Some(id) = environment {
ctx.db().get_environment_for_workspace(&workspace_id, id).map_err(|_| {
"Selected environment is missing or does not belong to this workspace".to_owned()
})?;
}
// Validate the entire selection before sending anything, including unchecked setup requests.
let mut enabled = 0;
for request in &requests {
yaak_assertions::validate(&request.assertions)
.map_err(|e| format!("{}: {e}", request.id))?;
enabled += request.assertions.checks.iter().filter(|c| c.enabled).count();
}
if enabled == 0 {
return Err("Selected requests have no enabled assertions".into());
}
if args.json.is_some() && args.json == args.junit {
return Err("JSON and JUnit reports require different files".into());
}
let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false);
let cancel_task = tokio::spawn(async move {
if tokio::signal::ctrl_c().await.is_ok() {
let _ = cancel_tx.send(true);
}
});
let plugin_context = PluginContext::new(Some("cli".into()), Some(workspace_id));
let response_dir = ctx.data_dir().join("responses");
let mut report = RunReport { version: 1, exit_code: 0, requests: Vec::new() };
let mut stopped = false;
for request in requests {
if stopped || *cancel_rx.borrow() {
report.requests.push(skipped(
&request.id,
&request.assertions,
if *cancel_rx.borrow() { "Canceled" } else { "Stopped after failure" },
));
continue;
}
let id = request.id.clone();
let definition = request.assertions.clone();
let result = send_http_request_with_plugins(SendHttpRequestWithPluginsParams {
query_manager: ctx.query_manager(),
blob_manager: ctx.blob_manager(),
request,
environment_id: environment,
update_source: UpdateSource::Sync,
cookie_jar_id: jar.clone(),
response_dir: &response_dir,
emit_events_to: None,
emit_response_body_chunks_to: None,
existing_response: None,
plugin_manager: ctx.plugin_manager(),
encryption_manager: ctx.encryption_manager.clone(),
plugin_context: &plugin_context,
cancelled_rx: Some(cancel_rx.clone()),
connection_manager: ctx.connection_manager(),
})
.await;
let request_result = match result {
Ok(result) if result.response.error.is_none() && !*cancel_rx.borrow() => {
summarize(&id, &definition, result.response.assertion_results.as_ref())
}
_ => RequestResult {
request_id: id,
outcome: "error".into(),
reason: if *cancel_rx.borrow() {
"Request canceled"
} else {
"Request did not complete successfully"
}
.into(),
checks: definition
.checks
.iter()
.map(|c| CheckResult {
assertion_id: c.id.clone(),
outcome: if c.enabled { "error" } else { "skipped" }.into(),
reason: if c.enabled {
"Request did not complete successfully"
} else {
"Disabled"
}
.into(),
})
.collect(),
},
};
if matches!(request_result.outcome.as_str(), "failed" | "error") {
report.exit_code = 1;
stopped = args.fail_fast;
}
report.requests.push(request_result);
}
if *cancel_rx.borrow() {
report.exit_code = 130;
}
cancel_task.abort();
Ok(report)
}
fn summarize(
id: &str,
definition: &HttpAssertions,
report: Option<&AssertionReport>,
) -> RequestResult {
let checks: Vec<CheckResult> = report
.map(|r| {
r.results
.iter()
.map(|c| CheckResult {
assertion_id: c.assertion_id.clone(),
outcome: c.outcome.clone(),
reason: c.reason.clone(),
})
.collect()
})
.unwrap_or_default();
let enabled = definition.checks.iter().filter(|c| c.enabled).count();
let (outcome, reason) = if report.is_some_and(|r| r.error.is_some())
|| checks.iter().any(|c| matches!(c.outcome.as_str(), "error" | "invalid"))
{
("error", "Assertions could not be evaluated")
} else if checks.iter().any(|c| c.outcome == "failed") {
("failed", "Assertions failed")
} else if enabled == 0 {
("unchecked", "No enabled assertions")
} else if checks.iter().filter(|c| c.outcome == "passed").count() != enabled {
("error", "Assertion results are incomplete")
} else {
("passed", "All enabled assertions passed")
};
RequestResult { request_id: id.into(), outcome: outcome.into(), reason: reason.into(), checks }
}
fn skipped(id: &str, definition: &HttpAssertions, reason: &str) -> RequestResult {
RequestResult {
request_id: id.into(),
outcome: "skipped".into(),
reason: reason.into(),
checks: definition
.checks
.iter()
.map(|c| CheckResult {
assertion_id: c.id.clone(),
outcome: "skipped".into(),
reason: reason.into(),
})
.collect(),
}
}
fn select_requests(ctx: &CliContext, id: &str) -> Result<(String, Vec<HttpRequest>), String> {
let db = ctx.db();
if let Ok(request) = db.get_any_request(id) {
return match request {
AnyRequest::HttpRequest(r) => Ok((r.workspace_id.clone(), vec![r])),
_ => Err("Assertions currently support HTTP requests only".into()),
};
}
let (workspace_id, parent) = if let Ok(folder) = db.get_folder(id) {
(folder.workspace_id, Some(folder.id))
} else if let Ok(workspace) = db.get_workspace(id) {
(workspace.id, None)
} else {
return Err("Could not resolve request, folder, or workspace".into());
};
let folders = db.list_folders(&workspace_id).map_err(|e| e.to_string())?;
let requests = db.list_http_requests(&workspace_id).map_err(|e| e.to_string())?;
// Refuse mixed protocols rather than silently producing a green partial suite.
let mut folder_ids = HashSet::new();
let ordered = ordered_requests(&folders, &requests, parent.as_deref(), &mut folder_ids)?;
let selected = |folder: Option<&String>| {
parent.is_none() || folder.is_some_and(|f| folder_ids.contains(f))
};
if db
.list_grpc_requests(&workspace_id)
.map_err(|e| e.to_string())?
.iter()
.any(|r| selected(r.folder_id.as_ref()))
|| db
.list_websocket_requests(&workspace_id)
.map_err(|e| e.to_string())?
.iter()
.any(|r| selected(r.folder_id.as_ref()))
{
return Err(
"Selection includes gRPC or WebSocket requests; choose an HTTP request or folder"
.into(),
);
}
if parent.is_none() && ordered.len() != requests.len() {
return Err("Request folder hierarchy is invalid".into());
}
Ok((workspace_id, ordered))
}
fn ordered_requests(
folders: &[Folder],
requests: &[HttpRequest],
parent: Option<&str>,
visited: &mut HashSet<String>,
) -> Result<Vec<HttpRequest>, String> {
if let Some(id) = parent {
if !visited.insert(id.to_owned()) {
return Err("Request folder hierarchy contains a cycle".into());
}
}
let mut children: Vec<_> = folders
.iter()
.filter(|f| f.folder_id.as_deref() == parent)
.map(|f| (f.sort_priority, f.created_at, f.id.as_str(), None::<&HttpRequest>))
.collect();
children.extend(
requests
.iter()
.filter(|r| r.folder_id.as_deref() == parent)
.map(|r| (r.sort_priority, r.created_at, r.id.as_str(), Some(r))),
);
children.sort_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1)).then(a.2.cmp(b.2)));
let mut ordered = Vec::new();
for (_, _, id, request) in children {
if let Some(request) = request {
ordered.push(request.clone());
} else {
ordered.extend(ordered_requests(folders, requests, Some(id), visited)?);
}
}
Ok(ordered)
}
fn write_reports(args: &TestArgs, report: &RunReport) -> Result<(), String> {
if let Some(path) = &args.json {
std::fs::write(path, serde_json::to_vec_pretty(report).map_err(|e| e.to_string())?)
.map_err(|e| e.to_string())?;
}
if let Some(path) = &args.junit {
std::fs::write(path, junit(report)).map_err(|e| e.to_string())?;
}
Ok(())
}
fn xml(s: &str) -> String {
s.chars()
.filter(|&c| {
matches!(c, '\t' | '\n' | '\r') || c >= ' ' && c != '\u{fffe}' && c != '\u{ffff}'
})
.collect::<String>()
.replace('&', "&amp;")
.replace('<', "&lt;")
.replace('>', "&gt;")
.replace('"', "&quot;")
.replace('\'', "&apos;")
}
fn junit(report: &RunReport) -> String {
let mut cases = Vec::new();
for request in &report.requests {
for check in &request.checks {
cases.push((
request.request_id.as_str(),
check.assertion_id.as_str(),
check.outcome.as_str(),
check.reason.as_str(),
));
}
if request.checks.is_empty()
|| request.outcome == "error"
&& !request.checks.iter().any(|c| c.outcome == "error" || c.outcome == "invalid")
{
cases.push((
request.request_id.as_str(),
"request",
if request.outcome == "unchecked" { "skipped" } else { &request.outcome },
request.reason.as_str(),
));
}
}
let count = |outcome| cases.iter().filter(|c| c.2 == outcome).count();
let mut out = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuite name=\"Yaak\" tests=\"{}\" failures=\"{}\" errors=\"{}\" skipped=\"{}\">\n",
cases.len(),
count("failed"),
count("error") + count("invalid"),
count("skipped")
);
for (request, check, outcome, reason) in cases {
out.push_str(&format!(
" <testcase classname=\"{}\" name=\"{}\">",
xml(request),
xml(check)
));
let tag = match outcome {
"failed" => Some("failure"),
"error" | "invalid" => Some("error"),
"skipped" => Some("skipped"),
_ => None,
};
if let Some(tag) = tag {
out.push_str(&format!("<{tag} message=\"{}\"/>", xml(reason)));
}
out.push_str("</testcase>\n");
}
out.push_str("</testsuite>\n");
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sidebar_order_interleaves_folders_and_requests() {
let folders =
vec![Folder { id: "folder".into(), sort_priority: 2.0, ..Default::default() }];
let requests = vec![
HttpRequest { id: "last".into(), sort_priority: 3.0, ..Default::default() },
HttpRequest {
id: "nested".into(),
folder_id: Some("folder".into()),
..Default::default()
},
HttpRequest { id: "first".into(), sort_priority: 1.0, ..Default::default() },
];
let ordered = ordered_requests(&folders, &requests, None, &mut HashSet::new()).unwrap();
assert_eq!(
ordered.iter().map(|r| r.id.as_str()).collect::<Vec<_>>(),
vec!["first", "nested", "last"]
);
}
#[test]
fn junit_escapes_attributes_and_counts_all_outcomes() {
let report = RunReport {
version: 1,
exit_code: 1,
requests: vec![
RequestResult {
request_id: "rq_<&\"".into(),
outcome: "failed".into(),
reason: "Assertions failed".into(),
checks: vec![CheckResult {
assertion_id: "check".into(),
outcome: "failed".into(),
reason: "Value did not match".into(),
}],
},
skipped("rq_skip", &HttpAssertions::default(), "Stopped after failure"),
],
};
let xml = junit(&report);
assert!(xml.contains("tests=\"2\" failures=\"1\" errors=\"0\" skipped=\"1\""));
assert!(xml.contains("classname=\"rq_&lt;&amp;&quot;\""));
assert!(xml.contains("<skipped "));
}
}
+26
View File
@@ -129,6 +129,32 @@ async fn main() {
}
}
}
Commands::Test(args) => {
let mut context = CliContext::new(data_dir.clone(), app_id);
match resolve_send_execution_context(
&context,
&args.id,
environment.as_deref(),
cookie_jar.as_deref(),
) {
Ok(execution_context) => {
context.init_plugins(execution_context).await;
let exit_code = commands::test::run(
&context,
args,
environment.as_deref(),
cookie_jar.as_deref(),
)
.await;
context.shutdown().await;
exit_code
}
Err(error) => {
eprintln!("Error: {error}");
2
}
}
}
Commands::CookieJar(args) => {
let context = CliContext::new(data_dir.clone(), app_id);
let exit_code = commands::cookie_jar::run(&context, args);
@@ -0,0 +1,164 @@
mod common;
use common::http_server::TestHttpServer;
use common::{cli_cmd, query_manager, seed_workspace};
use predicates::str::contains;
use tempfile::TempDir;
use yaak_models::models::{HttpAssertion, HttpAssertions, HttpRequest};
use yaak_models::util::UpdateSource;
fn request(id: &str, url: &str, expected: &str) -> HttpRequest {
HttpRequest {
id: id.into(),
workspace_id: "wk_test".into(),
url: url.into(),
name: "SECRET_REQUEST_NAME".into(),
assertions: HttpAssertions {
version: 1,
checks: vec![HttpAssertion {
id: "assertion".into(),
selector: "$.token".into(),
expected: expected.into(),
..Default::default()
}],
},
..Default::default()
}
}
fn save(dir: &std::path::Path, request: &HttpRequest) {
query_manager(dir).with_tx(|tx| tx.upsert_http_request(request, &UpdateSource::Sync)).unwrap();
}
#[test]
fn sends_assertions_persists_snapshots_and_writes_safe_reports() {
let dir = TempDir::new().unwrap();
seed_workspace(dir.path(), "wk_test");
let server = TestHttpServer::spawn_ok(r#"{"token":"SECRET_TOKEN"}"#);
let mut req = request("rq_test", &server.url, "SECRET_TOKEN");
save(dir.path(), &req);
cli_cmd(dir.path())
.args(["test", "rq_test", "--environment", "missing"])
.assert()
.code(2)
.stderr(contains("Selected environment"));
assert!(
query_manager(dir.path())
.connect()
.list_http_responses("wk_test", None)
.unwrap()
.is_empty()
);
let json = dir.path().join("results.json");
let junit = dir.path().join("results.xml");
let output = cli_cmd(dir.path())
.args(["test", "rq_test", "--json"])
.arg(&json)
.arg("--junit")
.arg(&junit)
.assert()
.success()
.stdout(contains("1 passed, 0 failed"))
.get_output()
.clone();
for text in [
String::from_utf8_lossy(&output.stdout).into_owned(),
std::fs::read_to_string(&json).unwrap(),
std::fs::read_to_string(&junit).unwrap(),
] {
assert!(!text.contains("SECRET"));
assert!(!text.contains(&server.url));
assert!(!text.contains("$.token"));
}
let report: serde_json::Value = serde_json::from_slice(&std::fs::read(json).unwrap()).unwrap();
assert_eq!(report["requests"][0]["checks"][0]["outcome"], "passed");
let manager = query_manager(dir.path());
let history = manager.connect().list_http_responses("wk_test", None).unwrap();
assert_eq!(history.len(), 1);
let original_id = history[0].id.clone();
assert_eq!(history[0].assertion_results.as_ref().unwrap().definition, req.assertions);
req.assertions.checks[0].expected = "DIFFERENT_SECRET".into();
save(dir.path(), &req);
assert_eq!(
manager
.connect()
.get_http_response(&original_id)
.unwrap()
.assertion_results
.unwrap()
.definition
.checks[0]
.expected,
"SECRET_TOKEN"
);
cli_cmd(dir.path()).args(["test", "rq_test"]).assert().code(1).stdout(contains("1 failed"));
// Ordinary send still prints the response body and succeeds even when a check fails.
cli_cmd(dir.path())
.args(["send", "rq_test"])
.assert()
.success()
.stdout(contains("SECRET_TOKEN"));
}
#[test]
fn invalid_suite_and_empty_suite_do_not_send_and_fail_fast_records_skips() {
let dir = TempDir::new().unwrap();
seed_workspace(dir.path(), "wk_test");
let server = TestHttpServer::spawn_ok(r#"{"token":"actual"}"#);
let mut first = request("rq_a", &server.url, "different");
first.sort_priority = 0.0;
let mut second = request("rq_b", &server.url, "actual");
second.sort_priority = 1.0;
second.assertions.checks[0].selector = "$[".into();
save(dir.path(), &first);
save(dir.path(), &second);
cli_cmd(dir.path())
.args(["test", "wk_test"])
.assert()
.code(2)
.stderr(contains("valid JSONPath"));
assert!(
query_manager(dir.path())
.connect()
.list_http_responses("wk_test", None)
.unwrap()
.is_empty()
);
second.assertions.checks[0].selector = "token".into();
save(dir.path(), &second);
let json = dir.path().join("results.json");
cli_cmd(dir.path())
.args(["test", "wk_test", "--fail-fast", "--json"])
.arg(&json)
.assert()
.code(1)
.stdout(contains("1 failed, 0 errors, 0 unchecked, 1 skipped"));
let report: serde_json::Value = serde_json::from_slice(&std::fs::read(json).unwrap()).unwrap();
assert_eq!(report["requests"][1]["requestId"], "rq_b");
assert_eq!(report["requests"][1]["outcome"], "skipped");
assert_eq!(
query_manager(dir.path()).connect().list_http_responses("wk_test", None).unwrap().len(),
1
);
first.assertions.checks.clear();
save(dir.path(), &first);
cli_cmd(dir.path())
.args(["test", "rq_a"])
.assert()
.code(2)
.stderr(contains("no enabled assertions"));
}
#[test]
fn transport_errors_are_distinct_from_failed_comparisons() {
let dir = TempDir::new().unwrap();
seed_workspace(dir.path(), "wk_test");
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
drop(listener);
save(dir.path(), &request("rq_error", &url, "SECRET"));
cli_cmd(dir.path())
.args(["test", "rq_error"])
.assert()
.code(1)
.stdout(contains("0 failed, 1 errors"));
}