// Round-trip harness: import a spec with the local importer plugin, send every // request through the real Yaak CLI pipeline against a Prism mock of the same // spec, and report Prism's validation verdict for each request. // // Prism independently validates each incoming request against the spec (paths, // required parameters, body schemas, security), so a violation here is an // importer bug found by a second OpenAPI implementation rather than a snapshot // of our own output. // // Usage: node tests/roundtrip.mjs [spec.yaml ...] // YAAK_BIN=/path/to/yaak overrides the CLI (defaults to the repo debug build, // which embeds the plugins vendored from this checkout). import { execFileSync, spawn } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import { fileURLToPath } from "node:url"; const here = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(here, "../../.."); const yaakBin = process.env.YAAK_BIN ?? path.join(repoRoot, "target/debug/yaak"); const specs = process.argv.length > 2 ? process.argv.slice(2) : [ path.join(here, "fixtures/petstore.yaml"), ...fs .readdirSync(path.join(here, "fixtures/real-world")) .filter((f) => f.endsWith(".yaml")) .map((f) => path.join(here, "fixtures/real-world", f)), ]; if (!fs.existsSync(yaakBin)) { console.error(`Yaak CLI not found at ${yaakBin}. Build it with: cargo build -p yaak-cli`); process.exit(2); } // Pinned so local runs and CI judge against the same validator const PRISM_PACKAGE = "@stoplight/prism-cli@5.15.11"; // Accepted spec-quality gray zones, not importer bugs. httpbin's required // `url` query parameter has no example, and an empty value is preferable to // inventing fake query data even though Prism counts it as missing. const KNOWN_FLAGS = new Set(["httpbin.yaml GET ${[baseUrl]}/redirect-to"]); function yaak(dataDir, args) { return execFileSync(yaakBin, ["--data-dir", dataDir, ...args], { encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], }); } function yaakJson(dataDir, args) { return JSON.parse(yaak(dataDir, args)); } function listIds(output) { return output .split("\n") .map((line) => line.match(/^(\w+_\w+) - /)?.[1]) .filter(Boolean); } async function startPrism(spec, port) { for (let attempt = 0; attempt < 20; attempt++, port++) { const prism = spawn( "npx", ["-y", PRISM_PACKAGE, "mock", "--errors", "-p", String(port), "-h", "127.0.0.1", spec], { stdio: ["ignore", "pipe", "pipe"] }, ); let log = ""; prism.stdout.on("data", (d) => (log += d)); prism.stderr.on("data", (d) => (log += d)); // Generous: the first run downloads Prism through npx const deadline = Date.now() + 120_000; let failed = false; while (Date.now() < deadline) { if (log.includes("Prism is listening")) return { prism, port, getLog: () => log }; if (log.includes("EADDRINUSE") || prism.exitCode != null) { failed = true; break; } await new Promise((r) => setTimeout(r, 200)); } prism.kill(); if (failed) { // The exit can be observed before its buffered stderr arrives; wait for // the streams to close so the bind error is distinguishable await Promise.race([ new Promise((r) => prism.once("close", r)), new Promise((r) => setTimeout(r, 2000)), ]); if (log.includes("EADDRINUSE")) continue; throw new Error(`Prism exited:\n${log}`); } throw new Error(`Prism did not start in time:\n${log}`); } throw new Error("No free port found for Prism"); } function pointVariablesAtPrism(variables, prismUrl) { return variables.map((v) => { if (v.name === "baseUrl" || v.name.startsWith("serverUrl")) return { ...v, value: prismUrl }; if (v.name === "baseUrlOrigin") return { ...v, value: prismUrl }; if (v.value === "") return { ...v, value: "test-value" }; return v; }); } // Prism reports its verdict in the sl-violations response header. Violations // located in the request are importer bugs; violations located in the response // mean Prism could not fabricate a spec-valid mock response (the spec's own // examples are broken), which says nothing about the import. function classify(response, bodyText) { if (response.error) return { verdict: "SEND ERROR", detail: response.error }; const violationsHeader = (response.headers ?? []).find( (h) => h.name?.toLowerCase() === "sl-violations", ); let violations = []; try { violations = JSON.parse(violationsHeader?.value ?? "[]"); } catch {} const requestViolations = violations.filter((v) => v.location?.[0] === "request"); const detail = requestViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; "); if (requestViolations.length > 0) return { verdict: "VIOLATION", detail }; // A spec-defined error response (e.g. an operation whose only response is a // 405) mocks as that status with no Prism error type; only Prism's own error // bodies mark a request Prism could not accept. let body = null; try { body = JSON.parse(bodyText); } catch {} const prismError = typeof body?.type === "string" && body.type.includes("stoplight.io/prism/errors") ? body.type.split("#")[1] : null; if (prismError == null) return { verdict: "ok", detail: "" }; // Failures to fabricate a mock response say nothing about the request we sent if (prismError === "NO_COMPLEX_OBJECT_TEXT" || prismError === "NO_RESPONSE_DEFINED") { return { verdict: "ok", detail: "" }; } const bodyViolations = Array.isArray(body.validation) ? body.validation.filter((v) => v.location?.[0] === "request") : []; if (prismError === "VIOLATIONS" && bodyViolations.length === 0) { return { verdict: "ok", detail: "" }; // response-side only } return { verdict: prismError === "VIOLATIONS" || prismError === "UNPROCESSABLE_ENTITY" ? "VIOLATION" : prismError.includes("MATCHED") ? "NO ROUTE" : prismError === "UNAUTHORIZED" ? "SECURITY" : prismError, detail: bodyViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ") || body.detail || "", }; } let totalProblems = 0; let port = 4010; for (const spec of specs) { const name = path.basename(spec); const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "yaak-roundtrip-")); console.log(`\n=== ${name} ===`); try { yaak(dataDir, ["import", spec]); const workspaceId = listIds(yaak(dataDir, ["workspace", "list"]))[0]; if (workspaceId == null) { console.log(" IMPORT PRODUCED NO WORKSPACE"); totalProblems++; continue; } // Prism mocks redirect responses without a Location header; following them // would fail the send for a reason unrelated to the import. Workspace-level // OAuth2 gets the same dummy-bearer treatment as request-level below. const workspace = yaakJson(dataDir, ["workspace", "show", workspaceId]); yaak(dataDir, [ "workspace", "update", "--json", JSON.stringify({ id: workspaceId, settingFollowRedirects: false, ...(workspace.authenticationType === "oauth2" ? { authenticationType: "bearer", authentication: { token: "test-token", prefix: "Bearer" }, } : {}), }), ]); const { prism, port: boundPort, getLog } = await startPrism(spec, port); port = boundPort + 1; try { const prismUrl = `http://127.0.0.1:${boundPort}`; const environmentIds = listIds(yaak(dataDir, ["environment", "list", workspaceId])); let activeEnvironment = null; for (const id of environmentIds) { const environment = yaakJson(dataDir, ["environment", "show", id]); yaak(dataDir, [ "environment", "update", "--json", JSON.stringify({ id, variables: pointVariablesAtPrism(environment.variables ?? [], prismUrl), }), ]); if (environment.parentModel === "environment" && activeEnvironment == null) { activeEnvironment = id; } } const requestIds = listIds(yaak(dataDir, ["request", "list", workspaceId])); const requests = new Map(); for (const id of requestIds) { const request = yaakJson(dataDir, ["request", "show", id]); requests.set(id, request); // OAuth2 would try to fetch a real token; Prism only checks that the // Authorization header is present, so a dummy bearer keeps it satisfied. if (request.authenticationType === "oauth2") { yaak(dataDir, [ "request", "update", "--json", JSON.stringify({ id, authenticationType: "bearer", authentication: { token: "test-token", prefix: "Bearer" }, }), ]); } } const sendArgs = ["send", workspaceId]; if (activeEnvironment != null) sendArgs.push("-e", activeEnvironment); try { yaak(dataDir, sendArgs); } catch { // Individual send failures surface per-request below. } let problems = 0; for (const id of requestIds) { const request = requests.get(id); const label = `${request.method} ${request.url}`; let response = null; let bodyText = ""; try { response = yaakJson(dataDir, ["response", "show", id]); try { bodyText = yaak(dataDir, ["response", "body", id]); } catch {} } catch { console.log(` NEVER SENT ${label}`); problems++; continue; } const { verdict, detail } = classify(response, bodyText); if (verdict === "ok") continue; if (KNOWN_FLAGS.has(`${name} ${label}`)) { console.log(` known ${label}`); continue; } problems++; console.log(` ${verdict.padEnd(12)} ${label}`); console.log(` sent: ${response.url ?? "?"}`); if (detail) console.log(` ${detail}`); } const requestCount = requestIds.length; if (problems === 0) { console.log(` all ${requestCount} requests validated clean against the mock`); } else { console.log(` ${problems}/${requestCount} requests flagged`); totalProblems += problems; } const inputWarnings = getLog() .split("\n") .filter((l) => l.includes("[VALIDATOR]") && !l.includes("output")); if (inputWarnings.length > 0) { console.log(` prism validator log lines: ${inputWarnings.length}`); } } finally { prism.kill(); } } finally { fs.rmSync(dataDir, { recursive: true, force: true }); } } process.exit(totalProblems > 0 ? 1 : 0);