mirror of
https://github.com/mountain-loop/yaak.git
synced 2026-08-21 02:44:12 +02:00
Add OpenAPI import round-trip harness (#600)
This commit is contained in:
Gregory Schier
@@ -31,3 +31,7 @@ jobs:
|
||||
run: vp test
|
||||
- name: Run Rust Tests
|
||||
run: cargo test --all --features yaak-app-client/wry
|
||||
- name: OpenAPI import round-trip
|
||||
run: |
|
||||
cargo build -p yaak-cli
|
||||
node plugins/importer-openapi/tests/roundtrip.mjs
|
||||
|
||||
@@ -7,7 +7,8 @@
|
||||
"scripts": {
|
||||
"build": "yaakcli build",
|
||||
"dev": "yaakcli dev",
|
||||
"test": "vp test --run tests"
|
||||
"test": "vp test --run tests",
|
||||
"test:roundtrip": "node tests/roundtrip.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"yaml": "^2.8.3"
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
// Round-trip harness: import a spec with the local importer plugin, send every
|
||||
// request through the real Yaak CLI pipeline against a Prism mock of the same
|
||||
// spec, and report Prism's validation verdict for each request.
|
||||
//
|
||||
// Prism independently validates each incoming request against the spec (paths,
|
||||
// required parameters, body schemas, security), so a violation here is an
|
||||
// importer bug found by a second OpenAPI implementation rather than a snapshot
|
||||
// of our own output.
|
||||
//
|
||||
// Usage: node tests/roundtrip.mjs [spec.yaml ...]
|
||||
// YAAK_BIN=/path/to/yaak overrides the CLI (defaults to the repo debug build,
|
||||
// which embeds the plugins vendored from this checkout).
|
||||
|
||||
import { execFileSync, spawn } from "node:child_process";
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(here, "../../..");
|
||||
const yaakBin = process.env.YAAK_BIN ?? path.join(repoRoot, "target/debug/yaak");
|
||||
const specs =
|
||||
process.argv.length > 2
|
||||
? process.argv.slice(2)
|
||||
: [
|
||||
path.join(here, "fixtures/petstore.yaml"),
|
||||
...fs
|
||||
.readdirSync(path.join(here, "fixtures/real-world"))
|
||||
.filter((f) => f.endsWith(".yaml"))
|
||||
.map((f) => path.join(here, "fixtures/real-world", f)),
|
||||
];
|
||||
|
||||
if (!fs.existsSync(yaakBin)) {
|
||||
console.error(`Yaak CLI not found at ${yaakBin}. Build it with: cargo build -p yaak-cli`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
// Pinned so local runs and CI judge against the same validator
|
||||
const PRISM_PACKAGE = "@stoplight/prism-cli@5.15.11";
|
||||
|
||||
// Accepted spec-quality gray zones, not importer bugs. httpbin's required
|
||||
// `url` query parameter has no example, and an empty value is preferable to
|
||||
// inventing fake query data even though Prism counts it as missing.
|
||||
const KNOWN_FLAGS = new Set(["httpbin.yaml GET ${[baseUrl]}/redirect-to"]);
|
||||
|
||||
function yaak(dataDir, args) {
|
||||
return execFileSync(yaakBin, ["--data-dir", dataDir, ...args], {
|
||||
encoding: "utf-8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
}
|
||||
|
||||
function yaakJson(dataDir, args) {
|
||||
return JSON.parse(yaak(dataDir, args));
|
||||
}
|
||||
|
||||
function listIds(output) {
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => line.match(/^(\w+_\w+) - /)?.[1])
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function startPrism(spec, port) {
|
||||
for (let attempt = 0; attempt < 20; attempt++, port++) {
|
||||
const prism = spawn(
|
||||
"npx",
|
||||
["-y", PRISM_PACKAGE, "mock", "--errors", "-p", String(port), "-h", "127.0.0.1", spec],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
let log = "";
|
||||
prism.stdout.on("data", (d) => (log += d));
|
||||
prism.stderr.on("data", (d) => (log += d));
|
||||
// Generous: the first run downloads Prism through npx
|
||||
const deadline = Date.now() + 120_000;
|
||||
let failed = false;
|
||||
while (Date.now() < deadline) {
|
||||
if (log.includes("Prism is listening")) return { prism, port, getLog: () => log };
|
||||
if (log.includes("EADDRINUSE") || prism.exitCode != null) {
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
}
|
||||
prism.kill();
|
||||
if (failed) {
|
||||
// The exit can be observed before its buffered stderr arrives; wait for
|
||||
// the streams to close so the bind error is distinguishable
|
||||
await Promise.race([
|
||||
new Promise((r) => prism.once("close", r)),
|
||||
new Promise((r) => setTimeout(r, 2000)),
|
||||
]);
|
||||
if (log.includes("EADDRINUSE")) continue;
|
||||
throw new Error(`Prism exited:\n${log}`);
|
||||
}
|
||||
throw new Error(`Prism did not start in time:\n${log}`);
|
||||
}
|
||||
throw new Error("No free port found for Prism");
|
||||
}
|
||||
|
||||
function pointVariablesAtPrism(variables, prismUrl) {
|
||||
return variables.map((v) => {
|
||||
if (v.name === "baseUrl" || v.name.startsWith("serverUrl")) return { ...v, value: prismUrl };
|
||||
if (v.name === "baseUrlOrigin") return { ...v, value: prismUrl };
|
||||
if (v.value === "") return { ...v, value: "test-value" };
|
||||
return v;
|
||||
});
|
||||
}
|
||||
|
||||
// Prism reports its verdict in the sl-violations response header. Violations
|
||||
// located in the request are importer bugs; violations located in the response
|
||||
// mean Prism could not fabricate a spec-valid mock response (the spec's own
|
||||
// examples are broken), which says nothing about the import.
|
||||
function classify(response, bodyText) {
|
||||
if (response.error) return { verdict: "SEND ERROR", detail: response.error };
|
||||
|
||||
const violationsHeader = (response.headers ?? []).find(
|
||||
(h) => h.name?.toLowerCase() === "sl-violations",
|
||||
);
|
||||
let violations = [];
|
||||
try {
|
||||
violations = JSON.parse(violationsHeader?.value ?? "[]");
|
||||
} catch {}
|
||||
const requestViolations = violations.filter((v) => v.location?.[0] === "request");
|
||||
const detail = requestViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ");
|
||||
|
||||
if (requestViolations.length > 0) return { verdict: "VIOLATION", detail };
|
||||
|
||||
// A spec-defined error response (e.g. an operation whose only response is a
|
||||
// 405) mocks as that status with no Prism error type; only Prism's own error
|
||||
// bodies mark a request Prism could not accept.
|
||||
let body = null;
|
||||
try {
|
||||
body = JSON.parse(bodyText);
|
||||
} catch {}
|
||||
const prismError =
|
||||
typeof body?.type === "string" && body.type.includes("stoplight.io/prism/errors")
|
||||
? body.type.split("#")[1]
|
||||
: null;
|
||||
if (prismError == null) return { verdict: "ok", detail: "" };
|
||||
|
||||
// Failures to fabricate a mock response say nothing about the request we sent
|
||||
if (prismError === "NO_COMPLEX_OBJECT_TEXT" || prismError === "NO_RESPONSE_DEFINED") {
|
||||
return { verdict: "ok", detail: "" };
|
||||
}
|
||||
|
||||
const bodyViolations = Array.isArray(body.validation)
|
||||
? body.validation.filter((v) => v.location?.[0] === "request")
|
||||
: [];
|
||||
if (prismError === "VIOLATIONS" && bodyViolations.length === 0) {
|
||||
return { verdict: "ok", detail: "" }; // response-side only
|
||||
}
|
||||
return {
|
||||
verdict:
|
||||
prismError === "VIOLATIONS" || prismError === "UNPROCESSABLE_ENTITY"
|
||||
? "VIOLATION"
|
||||
: prismError.includes("MATCHED")
|
||||
? "NO ROUTE"
|
||||
: prismError === "UNAUTHORIZED"
|
||||
? "SECURITY"
|
||||
: prismError,
|
||||
detail:
|
||||
bodyViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ") ||
|
||||
body.detail ||
|
||||
"",
|
||||
};
|
||||
}
|
||||
|
||||
let totalProblems = 0;
|
||||
let port = 4010;
|
||||
|
||||
for (const spec of specs) {
|
||||
const name = path.basename(spec);
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "yaak-roundtrip-"));
|
||||
console.log(`\n=== ${name} ===`);
|
||||
|
||||
try {
|
||||
yaak(dataDir, ["import", spec]);
|
||||
const workspaceId = listIds(yaak(dataDir, ["workspace", "list"]))[0];
|
||||
if (workspaceId == null) {
|
||||
console.log(" IMPORT PRODUCED NO WORKSPACE");
|
||||
totalProblems++;
|
||||
continue;
|
||||
}
|
||||
// Prism mocks redirect responses without a Location header; following them
|
||||
// would fail the send for a reason unrelated to the import. Workspace-level
|
||||
// OAuth2 gets the same dummy-bearer treatment as request-level below.
|
||||
const workspace = yaakJson(dataDir, ["workspace", "show", workspaceId]);
|
||||
yaak(dataDir, [
|
||||
"workspace",
|
||||
"update",
|
||||
"--json",
|
||||
JSON.stringify({
|
||||
id: workspaceId,
|
||||
settingFollowRedirects: false,
|
||||
...(workspace.authenticationType === "oauth2"
|
||||
? {
|
||||
authenticationType: "bearer",
|
||||
authentication: { token: "test-token", prefix: "Bearer" },
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
]);
|
||||
|
||||
const { prism, port: boundPort, getLog } = await startPrism(spec, port);
|
||||
port = boundPort + 1;
|
||||
try {
|
||||
const prismUrl = `http://127.0.0.1:${boundPort}`;
|
||||
const environmentIds = listIds(yaak(dataDir, ["environment", "list", workspaceId]));
|
||||
let activeEnvironment = null;
|
||||
for (const id of environmentIds) {
|
||||
const environment = yaakJson(dataDir, ["environment", "show", id]);
|
||||
yaak(dataDir, [
|
||||
"environment",
|
||||
"update",
|
||||
"--json",
|
||||
JSON.stringify({
|
||||
id,
|
||||
variables: pointVariablesAtPrism(environment.variables ?? [], prismUrl),
|
||||
}),
|
||||
]);
|
||||
if (environment.parentModel === "environment" && activeEnvironment == null) {
|
||||
activeEnvironment = id;
|
||||
}
|
||||
}
|
||||
|
||||
const requestIds = listIds(yaak(dataDir, ["request", "list", workspaceId]));
|
||||
const requests = new Map();
|
||||
for (const id of requestIds) {
|
||||
const request = yaakJson(dataDir, ["request", "show", id]);
|
||||
requests.set(id, request);
|
||||
// OAuth2 would try to fetch a real token; Prism only checks that the
|
||||
// Authorization header is present, so a dummy bearer keeps it satisfied.
|
||||
if (request.authenticationType === "oauth2") {
|
||||
yaak(dataDir, [
|
||||
"request",
|
||||
"update",
|
||||
"--json",
|
||||
JSON.stringify({
|
||||
id,
|
||||
authenticationType: "bearer",
|
||||
authentication: { token: "test-token", prefix: "Bearer" },
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
const sendArgs = ["send", workspaceId];
|
||||
if (activeEnvironment != null) sendArgs.push("-e", activeEnvironment);
|
||||
try {
|
||||
yaak(dataDir, sendArgs);
|
||||
} catch {
|
||||
// Individual send failures surface per-request below.
|
||||
}
|
||||
|
||||
let problems = 0;
|
||||
for (const id of requestIds) {
|
||||
const request = requests.get(id);
|
||||
const label = `${request.method} ${request.url}`;
|
||||
let response = null;
|
||||
let bodyText = "";
|
||||
try {
|
||||
response = yaakJson(dataDir, ["response", "show", id]);
|
||||
try {
|
||||
bodyText = yaak(dataDir, ["response", "body", id]);
|
||||
} catch {}
|
||||
} catch {
|
||||
console.log(` NEVER SENT ${label}`);
|
||||
problems++;
|
||||
continue;
|
||||
}
|
||||
const { verdict, detail } = classify(response, bodyText);
|
||||
if (verdict === "ok") continue;
|
||||
if (KNOWN_FLAGS.has(`${name} ${label}`)) {
|
||||
console.log(` known ${label}`);
|
||||
continue;
|
||||
}
|
||||
problems++;
|
||||
console.log(` ${verdict.padEnd(12)} ${label}`);
|
||||
console.log(` sent: ${response.url ?? "?"}`);
|
||||
if (detail) console.log(` ${detail}`);
|
||||
}
|
||||
|
||||
const requestCount = requestIds.length;
|
||||
if (problems === 0) {
|
||||
console.log(` all ${requestCount} requests validated clean against the mock`);
|
||||
} else {
|
||||
console.log(` ${problems}/${requestCount} requests flagged`);
|
||||
totalProblems += problems;
|
||||
}
|
||||
const inputWarnings = getLog()
|
||||
.split("\n")
|
||||
.filter((l) => l.includes("[VALIDATOR]") && !l.includes("output"));
|
||||
if (inputWarnings.length > 0) {
|
||||
console.log(` prism validator log lines: ${inputWarnings.length}`);
|
||||
}
|
||||
} finally {
|
||||
prism.kill();
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(totalProblems > 0 ? 1 : 0);
|
||||
Reference in New Issue
Block a user