Add OpenAPI import round-trip harness

plugins/importer-openapi/package.json

@@ -7,7 +7,8 @@
   "scripts": {
     "build": "yaakcli build",
     "dev": "yaakcli dev",
-    "test": "vp test --run tests"
+    "test": "vp test --run tests",
+    "test:roundtrip": "node tests/roundtrip.mjs"
   },
   "dependencies": {
     "yaml": "^2.8.3"

plugins/importer-openapi/tests/roundtrip.mjs

@@ -0,0 +1,265 @@
+// Round-trip harness: import a spec with the local importer plugin, send every
+// request through the real Yaak CLI pipeline against a Prism mock of the same
+// spec, and report Prism's validation verdict for each request.
+//
+// Prism independently validates each incoming request against the spec (paths,
+// required parameters, body schemas, security), so a violation here is an
+// importer bug found by a second OpenAPI implementation rather than a snapshot
+// of our own output.
+//
+// Usage: node tests/roundtrip.mjs [spec.yaml ...]
+//   YAAK_BIN=/path/to/yaak overrides the CLI (defaults to the repo debug build,
+//   which embeds the plugins vendored from this checkout).
+
+import { execFileSync, spawn } from "node:child_process";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.resolve(here, "../../..");
+const yaakBin = process.env.YAAK_BIN ?? path.join(repoRoot, "target/debug/yaak");
+const specs =
+  process.argv.length > 2
+    ? process.argv.slice(2)
+    : [
+        path.join(here, "fixtures/petstore.yaml"),
+        ...fs
+          .readdirSync(path.join(here, "fixtures/real-world"))
+          .filter((f) => f.endsWith(".yaml"))
+          .map((f) => path.join(here, "fixtures/real-world", f)),
+      ];
+
+if (!fs.existsSync(yaakBin)) {
+  console.error(`Yaak CLI not found at ${yaakBin}. Build it with: cargo build -p yaak-cli`);
+  process.exit(2);
+}
+
+function yaak(dataDir, args) {
+  return execFileSync(yaakBin, ["--data-dir", dataDir, ...args], {
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+}
+
+function yaakJson(dataDir, args) {
+  return JSON.parse(yaak(dataDir, args));
+}
+
+function listIds(output) {
+  return output
+    .split("\n")
+    .map((line) => line.match(/^(\w+_\w+) - /)?.[1])
+    .filter(Boolean);
+}
+
+async function startPrism(spec, port) {
+  const prism = spawn(
+    "npx",
+    ["-y", "@stoplight/prism-cli", "mock", "--errors", "-p", String(port), "-h", "127.0.0.1", spec],
+    { stdio: ["ignore", "pipe", "pipe"] },
+  );
+  let log = "";
+  prism.stdout.on("data", (d) => (log += d));
+  prism.stderr.on("data", (d) => (log += d));
+  const deadline = Date.now() + 30_000;
+  while (Date.now() < deadline) {
+    if (log.includes("Prism is listening")) return { prism, getLog: () => log };
+    if (prism.exitCode != null) throw new Error(`Prism exited:\n${log}`);
+    await new Promise((r) => setTimeout(r, 200));
+  }
+  prism.kill();
+  throw new Error(`Prism did not start in time:\n${log}`);
+}
+
+function pointVariablesAtPrism(variables, prismUrl) {
+  return variables.map((v) => {
+    if (v.name === "baseUrl" || v.name.startsWith("serverUrl")) return { ...v, value: prismUrl };
+    if (v.name === "baseUrlOrigin") return { ...v, value: prismUrl };
+    if (v.value === "") return { ...v, value: "test-value" };
+    return v;
+  });
+}
+
+// Prism reports its verdict in the sl-violations response header. Violations
+// located in the request are importer bugs; violations located in the response
+// mean Prism could not fabricate a spec-valid mock response (the spec's own
+// examples are broken), which says nothing about the import.
+function classify(response, bodyText) {
+  if (response.error) return { verdict: "SEND ERROR", detail: response.error };
+
+  const violationsHeader = (response.headers ?? []).find(
+    (h) => h.name?.toLowerCase() === "sl-violations",
+  );
+  let violations = [];
+  try {
+    violations = JSON.parse(violationsHeader?.value ?? "[]");
+  } catch {}
+  const requestViolations = violations.filter((v) => v.location?.[0] === "request");
+  const detail = requestViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ");
+
+  if (requestViolations.length > 0) return { verdict: "VIOLATION", detail };
+
+  // A spec-defined error response (e.g. an operation whose only response is a
+  // 405) mocks as that status with no Prism error type; only Prism's own error
+  // bodies mark a request Prism could not accept.
+  let body = null;
+  try {
+    body = JSON.parse(bodyText);
+  } catch {}
+  const prismError =
+    typeof body?.type === "string" && body.type.includes("stoplight.io/prism/errors")
+      ? body.type.split("#")[1]
+      : null;
+  if (prismError == null) return { verdict: "ok", detail: "" };
+
+  // Failures to fabricate a mock response say nothing about the request we sent
+  if (prismError === "NO_COMPLEX_OBJECT_TEXT" || prismError === "NO_RESPONSE_DEFINED") {
+    return { verdict: "ok", detail: "" };
+  }
+
+  const bodyViolations = Array.isArray(body.validation)
+    ? body.validation.filter((v) => v.location?.[0] === "request")
+    : [];
+  if (prismError === "VIOLATIONS" && bodyViolations.length === 0) {
+    return { verdict: "ok", detail: "" }; // response-side only
+  }
+  return {
+    verdict:
+      prismError === "VIOLATIONS" || prismError === "UNPROCESSABLE_ENTITY"
+        ? "VIOLATION"
+        : prismError.includes("MATCHED")
+          ? "NO ROUTE"
+          : prismError === "UNAUTHORIZED"
+            ? "SECURITY"
+            : prismError,
+    detail:
+      bodyViolations.map((v) => `${v.location.join(".")}: ${v.message}`).join("; ") ||
+      body.detail ||
+      "",
+  };
+}
+
+let totalProblems = 0;
+let port = 4010;
+
+for (const spec of specs) {
+  const name = path.basename(spec);
+  const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "yaak-roundtrip-"));
+  console.log(`\n=== ${name} ===`);
+
+  try {
+    yaak(dataDir, ["import", spec]);
+    const workspaceId = listIds(yaak(dataDir, ["workspace", "list"]))[0];
+    if (workspaceId == null) {
+      console.log("  IMPORT PRODUCED NO WORKSPACE");
+      totalProblems++;
+      continue;
+    }
+    // Prism mocks redirect responses without a Location header; following them
+    // would fail the send for a reason unrelated to the import.
+    yaak(dataDir, [
+      "workspace",
+      "update",
+      "--json",
+      JSON.stringify({ id: workspaceId, settingFollowRedirects: false }),
+    ]);
+
+    const prismUrl = `http://127.0.0.1:${port}`;
+    const environmentIds = listIds(yaak(dataDir, ["environment", "list", workspaceId]));
+    let activeEnvironment = null;
+    for (const id of environmentIds) {
+      const environment = yaakJson(dataDir, ["environment", "show", id]);
+      yaak(dataDir, [
+        "environment",
+        "update",
+        "--json",
+        JSON.stringify({
+          id,
+          variables: pointVariablesAtPrism(environment.variables ?? [], prismUrl),
+        }),
+      ]);
+      if (environment.parentModel === "environment" && activeEnvironment == null) {
+        activeEnvironment = id;
+      }
+    }
+
+    const requestIds = listIds(yaak(dataDir, ["request", "list", workspaceId]));
+    const requests = new Map();
+    for (const id of requestIds) {
+      const request = yaakJson(dataDir, ["request", "show", id]);
+      requests.set(id, request);
+      // OAuth2 would try to fetch a real token; Prism only checks that the
+      // Authorization header is present, so a dummy bearer keeps it satisfied.
+      if (request.authenticationType === "oauth2") {
+        yaak(dataDir, [
+          "request",
+          "update",
+          "--json",
+          JSON.stringify({
+            id,
+            authenticationType: "bearer",
+            authentication: { token: "test-token", prefix: "Bearer" },
+          }),
+        ]);
+      }
+    }
+
+    const { prism, getLog } = await startPrism(spec, port);
+    port++;
+    try {
+      const sendArgs = ["send", workspaceId];
+      if (activeEnvironment != null) sendArgs.push("-e", activeEnvironment);
+      try {
+        yaak(dataDir, sendArgs);
+      } catch {
+        // Individual send failures surface per-request below.
+      }
+
+      let problems = 0;
+      for (const id of requestIds) {
+        const request = requests.get(id);
+        const label = `${request.method} ${request.url}`;
+        let response = null;
+        let bodyText = "";
+        try {
+          response = yaakJson(dataDir, ["response", "show", id]);
+          try {
+            bodyText = yaak(dataDir, ["response", "body", id]);
+          } catch {}
+        } catch {
+          console.log(`  NEVER SENT   ${label}`);
+          problems++;
+          continue;
+        }
+        const { verdict, detail } = classify(response, bodyText);
+        if (verdict === "ok") continue;
+        problems++;
+        console.log(`  ${verdict.padEnd(12)} ${label}`);
+        console.log(`               sent: ${response.url ?? "?"}`);
+        if (detail) console.log(`               ${detail}`);
+      }
+
+      const requestCount = requestIds.length;
+      if (problems === 0) {
+        console.log(`  all ${requestCount} requests validated clean against the mock`);
+      } else {
+        console.log(`  ${problems}/${requestCount} requests flagged`);
+        totalProblems += problems;
+      }
+      const inputWarnings = getLog()
+        .split("\n")
+        .filter((l) => l.includes("[VALIDATOR]") && !l.includes("output"));
+      if (inputWarnings.length > 0) {
+        console.log(`  prism validator log lines: ${inputWarnings.length}`);
+      }
+    } finally {
+      prism.kill();
+    }
+  } finally {
+    fs.rmSync(dataDir, { recursive: true, force: true });
+  }
+}
+
+process.exit(totalProblems > 0 ? 1 : 0);