diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecf835e9..dd4f9082 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,3 +31,7 @@ jobs: run: vp test - name: Run Rust Tests run: cargo test --all --features yaak-app-client/wry + - name: OpenAPI import round-trip + run: | + cargo build -p yaak-cli + node plugins/importer-openapi/tests/roundtrip.mjs diff --git a/plugins/importer-openapi/package.json b/plugins/importer-openapi/package.json index e312db11..6d27c320 100644 --- a/plugins/importer-openapi/package.json +++ b/plugins/importer-openapi/package.json @@ -7,7 +7,8 @@ "scripts": { "build": "yaakcli build", "dev": "yaakcli dev", - "test": "vp test --run tests" + "test": "vp test --run tests", + "test:roundtrip": "node tests/roundtrip.mjs" }, "dependencies": { "yaml": "^2.8.3" diff --git a/plugins/importer-openapi/tests/roundtrip.mjs b/plugins/importer-openapi/tests/roundtrip.mjs new file mode 100644 index 00000000..5b3637ce --- /dev/null +++ b/plugins/importer-openapi/tests/roundtrip.mjs @@ -0,0 +1,306 @@ +// Round-trip harness: import a spec with the local importer plugin, send every +// request through the real Yaak CLI pipeline against a Prism mock of the same +// spec, and report Prism's validation verdict for each request. +// +// Prism independently validates each incoming request against the spec (paths, +// required parameters, body schemas, security), so a violation here is an +// importer bug found by a second OpenAPI implementation rather than a snapshot +// of our own output. +// +// Usage: node tests/roundtrip.mjs [spec.yaml ...] +// YAAK_BIN=/path/to/yaak overrides the CLI (defaults to the repo debug build, +// which embeds the plugins vendored from this checkout). + +import { execFileSync, spawn } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, "../../.."); +const yaakBin = process.env.YAAK_BIN ?? path.join(repoRoot, "target/debug/yaak"); +const specs = + process.argv.length > 2 + ? process.argv.slice(2) + : [ + path.join(here, "fixtures/petstore.yaml"), + ...fs + .readdirSync(path.join(here, "fixtures/real-world")) + .filter((f) => f.endsWith(".yaml")) + .map((f) => path.join(here, "fixtures/real-world", f)), + ]; + +if (!fs.existsSync(yaakBin)) { + console.error(`Yaak CLI not found at ${yaakBin}. Build it with: cargo build -p yaak-cli`); + process.exit(2); +} + +// Pinned so local runs and CI judge against the same validator +const PRISM_PACKAGE = "@stoplight/prism-cli@5.15.11"; + +// Accepted spec-quality gray zones, not importer bugs. httpbin's required +// `url` query parameter has no example, and an empty value is preferable to +// inventing fake query data even though Prism counts it as missing. +const KNOWN_FLAGS = new Set(["httpbin.yaml GET ${[baseUrl]}/redirect-to"]); + +function yaak(dataDir, args) { + return execFileSync(yaakBin, ["--data-dir", dataDir, ...args], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function yaakJson(dataDir, args) { + return JSON.parse(yaak(dataDir, args)); +} + +function listIds(output) { + return output + .split("\n") + .map((line) => line.match(/^(\w+_\w+) - /)?.[1]) + .filter(Boolean); +} + +async function startPrism(spec, port) { + for (let attempt = 0; attempt < 20; attempt++, port++) { + const prism = spawn( + "npx", + ["-y", PRISM_PACKAGE, "mock", "--errors", "-p", String(port), "-h", "127.0.0.1", spec], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + let log = ""; + prism.stdout.on("data", (d) => (log += d)); + prism.stderr.on("data", (d) => (log += d)); + // Generous: the first run downloads Prism through npx + const deadline = Date.now() + 120_000; + let failed = false; + while (Date.now() < deadline) { + if (log.includes("Prism is listening")) return { prism, port, getLog: () => log }; + if (log.includes("EADDRINUSE") || prism.exitCode != null) { + failed = true; + break; + } + await new Promise((r) => setTimeout(r, 200)); + } + prism.kill(); + if (failed) { + // The exit can be observed before its buffered stderr arrives; wait for + // the streams to close so the bind error is distinguishable + await Promise.race([ + new Promise((r) => prism.once("close", r)), + new Promise((r) => setTimeout(r, 2000)), + ]); + if (log.includes("EADDRINUSE")) continue; + throw new Error(`Prism exited:\n${log}`); + } + throw new Error(`Prism did not start in time:\n${log}`); + } + throw new Error("No free port found for Prism"); +} + +function pointVariablesAtPrism(variables, prismUrl) { + return variables.map((v) => { + if (v.name === "baseUrl" || v.name.startsWith("serverUrl")) return { ...v, value: prismUrl }; + if (v.name === "baseUrlOrigin") return { ...v, value: prismUrl }; + if (v.value === "") return { ...v, value: "test-value" }; + return v; + }); +} + +// Prism reports its verdict in the sl-violations response header. Violations +// located in the request are importer bugs; violations located in the response +// mean Prism could not fabricate a spec-valid mock response (the spec's own +// examples are broken), which says nothing about the import. +function classify(response, bodyText) { + if (response.error) return { verdict: "SEND ERROR", detail: response.error }; + + const violationsHeader = (response.headers ?? []).find( + (h) => h.name?.toLowerCase() === "sl-violations", + ); + let violations = []; + try { + violations = JSON.parse(violationsHeader?.value ?? "[]"); + } catch {} + const requestViolations = violations.filter((v) => v.location?.[0] === "request"); + const detail = requestViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; "); + + if (requestViolations.length > 0) return { verdict: "VIOLATION", detail }; + + // A spec-defined error response (e.g. an operation whose only response is a + // 405) mocks as that status with no Prism error type; only Prism's own error + // bodies mark a request Prism could not accept. + let body = null; + try { + body = JSON.parse(bodyText); + } catch {} + const prismError = + typeof body?.type === "string" && body.type.includes("stoplight.io/prism/errors") + ? body.type.split("#")[1] + : null; + if (prismError == null) return { verdict: "ok", detail: "" }; + + // Failures to fabricate a mock response say nothing about the request we sent + if (prismError === "NO_COMPLEX_OBJECT_TEXT" || prismError === "NO_RESPONSE_DEFINED") { + return { verdict: "ok", detail: "" }; + } + + const bodyViolations = Array.isArray(body.validation) + ? body.validation.filter((v) => v.location?.[0] === "request") + : []; + if (prismError === "VIOLATIONS" && bodyViolations.length === 0) { + return { verdict: "ok", detail: "" }; // response-side only + } + return { + verdict: + prismError === "VIOLATIONS" || prismError === "UNPROCESSABLE_ENTITY" + ? "VIOLATION" + : prismError.includes("MATCHED") + ? "NO ROUTE" + : prismError === "UNAUTHORIZED" + ? "SECURITY" + : prismError, + detail: + bodyViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ") || + body.detail || + "", + }; +} + +let totalProblems = 0; +let port = 4010; + +for (const spec of specs) { + const name = path.basename(spec); + const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "yaak-roundtrip-")); + console.log(`\n=== ${name} ===`); + + try { + yaak(dataDir, ["import", spec]); + const workspaceId = listIds(yaak(dataDir, ["workspace", "list"]))[0]; + if (workspaceId == null) { + console.log(" IMPORT PRODUCED NO WORKSPACE"); + totalProblems++; + continue; + } + // Prism mocks redirect responses without a Location header; following them + // would fail the send for a reason unrelated to the import. Workspace-level + // OAuth2 gets the same dummy-bearer treatment as request-level below. + const workspace = yaakJson(dataDir, ["workspace", "show", workspaceId]); + yaak(dataDir, [ + "workspace", + "update", + "--json", + JSON.stringify({ + id: workspaceId, + settingFollowRedirects: false, + ...(workspace.authenticationType === "oauth2" + ? { + authenticationType: "bearer", + authentication: { token: "test-token", prefix: "Bearer" }, + } + : {}), + }), + ]); + + const { prism, port: boundPort, getLog } = await startPrism(spec, port); + port = boundPort + 1; + try { + const prismUrl = `http://127.0.0.1:${boundPort}`; + const environmentIds = listIds(yaak(dataDir, ["environment", "list", workspaceId])); + let activeEnvironment = null; + for (const id of environmentIds) { + const environment = yaakJson(dataDir, ["environment", "show", id]); + yaak(dataDir, [ + "environment", + "update", + "--json", + JSON.stringify({ + id, + variables: pointVariablesAtPrism(environment.variables ?? [], prismUrl), + }), + ]); + if (environment.parentModel === "environment" && activeEnvironment == null) { + activeEnvironment = id; + } + } + + const requestIds = listIds(yaak(dataDir, ["request", "list", workspaceId])); + const requests = new Map(); + for (const id of requestIds) { + const request = yaakJson(dataDir, ["request", "show", id]); + requests.set(id, request); + // OAuth2 would try to fetch a real token; Prism only checks that the + // Authorization header is present, so a dummy bearer keeps it satisfied. + if (request.authenticationType === "oauth2") { + yaak(dataDir, [ + "request", + "update", + "--json", + JSON.stringify({ + id, + authenticationType: "bearer", + authentication: { token: "test-token", prefix: "Bearer" }, + }), + ]); + } + } + + const sendArgs = ["send", workspaceId]; + if (activeEnvironment != null) sendArgs.push("-e", activeEnvironment); + try { + yaak(dataDir, sendArgs); + } catch { + // Individual send failures surface per-request below. + } + + let problems = 0; + for (const id of requestIds) { + const request = requests.get(id); + const label = `${request.method} ${request.url}`; + let response = null; + let bodyText = ""; + try { + response = yaakJson(dataDir, ["response", "show", id]); + try { + bodyText = yaak(dataDir, ["response", "body", id]); + } catch {} + } catch { + console.log(` NEVER SENT ${label}`); + problems++; + continue; + } + const { verdict, detail } = classify(response, bodyText); + if (verdict === "ok") continue; + if (KNOWN_FLAGS.has(`${name} ${label}`)) { + console.log(` known ${label}`); + continue; + } + problems++; + console.log(` ${verdict.padEnd(12)} ${label}`); + console.log(` sent: ${response.url ?? "?"}`); + if (detail) console.log(` ${detail}`); + } + + const requestCount = requestIds.length; + if (problems === 0) { + console.log(` all ${requestCount} requests validated clean against the mock`); + } else { + console.log(` ${problems}/${requestCount} requests flagged`); + totalProblems += problems; + } + const inputWarnings = getLog() + .split("\n") + .filter((l) => l.includes("[VALIDATOR]") && !l.includes("output")); + if (inputWarnings.length > 0) { + console.log(` prism validator log lines: ${inputWarnings.length}`); + } + } finally { + prism.kill(); + } + } finally { + fs.rmSync(dataDir, { recursive: true, force: true }); + } +} + +process.exit(totalProblems > 0 ? 1 : 0);