From dafd16f4479f35ecbf649fda100cd8358320a1a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= <ryan4yin@linux.com>
Date: Sat, 3 Oct 2026 10:06:22 +0800
Subject: [PATCH] fix(ci): fail the eval-test run when the suite does not pass
 (#350)

* fix(ci): fail the eval-test run when the suite does not pass

`nix eval .#evalTests` prints the suite result but exits 0 either way, so
both `just test` and the CI workflow reported success even when a test
failed. Compare the raw boolean and fail instead.

* fix(ci): use --json for the boolean eval result

`nix eval --raw` cannot coerce a Boolean to a string, and Just runs each
recipe line as a separate `nu -c`, so the result variable must stay on one
line.
---
 .github/workflows/flake_evaltests.yml | 6 +++---
 Justfile                              | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/flake_evaltests.yml b/.github/workflows/flake_evaltests.yml
index 1b868966..e76b7204 100644
--- a/.github/workflows/flake_evaltests.yml
+++ b/.github/workflows/flake_evaltests.yml
@@ -37,6 +37,6 @@ jobs:
       - name: Run Nix Flake Eval Tests
         run: |
           echo 'Flake Eval Tests'
-          # stack overflow...
-          # nix eval .#checks --show-trace --print-build-logs --verbose
-          nix eval .#evalTests --show-trace --print-build-logs --verbose
+          # `nix eval` prints the suite result but still exits 0 on failure, so
+          # compare the JSON boolean explicitly to fail the job.
+          test "$(nix eval .#evalTests --json --show-trace)" = "true"
diff --git a/Justfile b/Justfile
index 138acd6f..3f214486 100644
--- a/Justfile
+++ b/Justfile
@@ -21,7 +21,7 @@ default:
 # Run eval tests
 [group('nix')]
 test:
-  nix eval .#evalTests --show-trace --print-build-logs --verbose
+  let result = (nix eval .#evalTests --json --show-trace --print-build-logs --verbose | str trim); if $result != "true" { error make { msg: $"eval tests failed: evalTests returned ($result)" } }
 
 # Evaluate a NixOS host configuration without building it.
 [group('nix')]