fix(ci): draw the host of a VM suite again when it has no KVM, in the same run

A hosted runner of the ubuntu-24.04 pool is not promised nested virtualization, and
3 of the 16 suites of run 42182554717 failed the check of /dev/kvm. rerun-no-kvm.yaml
could not help: a workflow_run workflow runs only from the default branch, where the
file is not, and a run cannot rerun its own failed jobs while it is in progress.

The step "Enable KVM" of the nix action is now the only one that enables KVM
(enable_kvm of install-nix-action is off), runs before the disk and the install,
waits for udev to settle and polls /dev/kvm for 30 seconds instead of testing it
once, 42 ms after an asynchronous udevadm trigger, and prints what tells a missing
device from a rule that was not applied yet. With defer-no-kvm a host without KVM
is an output, not a failure.

_vm.yaml calls _vm-attempt.yaml up to three times: the suites that found no KVM are
listed through the artifacts of the run and run again on new machines, and the last
attempt fails them. vm-fast and vm-slow are calls of it. rerun-no-kvm.yaml is
removed.

.github/actions/nix/action.yaml

@@ -21,6 +21,9 @@ inputs:
   kvm:
     description: Make /dev/kvm usable and tell nix that this machine can run NixOS tests
     default: "false"
+  defer-no-kvm:
+    description: "With kvm, a host without a usable /dev/kvm does not fail the step: the output kvm is `missing`, every other step of the action is skipped, and the caller runs the job again on another host (.github/workflows/_vm.yaml). Off, that host fails the step"
+    default: "false"
   free-disk:
     description: Delete the toolchains of the hosted image that this repository does not use (about 25 GB)
     default: "false"
@@ -37,20 +40,86 @@ inputs:
     description: The token that may push to the cache, secrets.CACHIX_AUTH_TOKEN
     default: ""
 
+outputs:
+  kvm:
+    description: "`ready` (kvm asked for and /dev/kvm is usable), `missing` (asked for with defer-no-kvm, and the host has none), empty (not asked for)"
+    value: ${{ steps.kvm.outputs.state }}
+
 runs:
   using: composite
   steps:
+    # The udev rule is what makes /dev/kvm writable for the runner's user (the one GitHub documents:
+    # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ ).
+    # `udevadm trigger` returns before the rule is applied, so the device is polled, for 30 seconds at
+    # most, and not tested once. The step comes first: a host without KVM is known in a second and not
+    # after the disk and the nix install. cachix/install-nix-action writes the same rule and ignores the
+    # outcome (`KVM is not available`), so its enable_kvm is off below: this step is the only one.
+    #
+    # The caller can tell the failure of this step from a failed suite: it is the step "Run
+    # ./.github/actions/nix" of the job (the API lists a composite action as one step), and with
+    # defer-no-kvm it is not a failure at all but the output kvm=missing.
+    - name: Enable KVM
+      id: kvm
+      if: inputs.kvm == 'true'
+      shell: bash
+      env:
+        DEFER: ${{ inputs.defer-no-kvm }}
+      run: |
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' |
+          sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        diagnose() {
+          echo "::group::KVM diagnostics ($1)"
+          ls -l /dev/kvm 2>&1 || true
+          id
+          getent group kvm || true
+          systemd-detect-virt || true
+          echo "cpu flags vmx/svm: $(grep -E -c 'vmx|svm' /proc/cpuinfo || true)"
+          lsmod | grep -E '^kvm' || echo "no kvm module listed"
+          echo "::endgroup::"
+        }
+        diagnose "before the rule"
+        # a module that is not loaded is the one case that the rule cannot fix; harmless where it is
+        sudo modprobe -q kvm_intel 2>/dev/null || sudo modprobe -q kvm_amd 2>/dev/null || true
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger --name-match=kvm
+        sudo udevadm settle --timeout=30 || true
+        ready=false
+        for second in $(seq 0 30); do
+          if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then
+            echo "/dev/kvm is usable after $second s"
+            ready=true
+            break
+          fi
+          sleep 1
+        done
+        if [ "$ready" = false ] && [ -e /dev/kvm ]; then
+          # the device is there and the rule did not reach it: set the mode by hand
+          echo "/dev/kvm exists and is not usable after 30 s: udev did not apply the rule, setting the mode by hand"
+          sudo chmod 0666 /dev/kvm && [ -r /dev/kvm ] && [ -w /dev/kvm ] && ready=true
+        fi
+        diagnose "after"
+        if [ "$ready" = true ]; then
+          echo "state=ready" >> "$GITHUB_OUTPUT"
+        elif [ "$DEFER" = true ]; then
+          echo "state=missing" >> "$GITHUB_OUTPUT"
+          echo "::warning::this host has no usable /dev/kvm ($([ -e /dev/kvm ] || echo 'the device is absent: no nested virtualization'))"
+        else
+          # the host of a hosted runner is not chosen: some have no nested virtualization
+          echo "::error::this runner has no usable /dev/kvm"
+          exit 1
+        fi
+
     # A Plasma guest is a closure of about 10 GB, and a hosted runner has little free space on /
     # until the toolchains it ships with are gone.
     - name: Free disk space
-      if: inputs.free-disk == 'true' || inputs.kvm == 'true'
+      if: steps.kvm.outputs.state != 'missing' && (inputs.free-disk == 'true' || inputs.kvm == 'true')
       shell: bash
       run: |
         sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
         df -h /
 
     - name: Add swap
-      if: inputs.swap == 'true'
+      if: steps.kvm.outputs.state != 'missing' && inputs.swap == 'true'
       shell: bash
       run: |
         sudo swapoff -a || true
@@ -60,27 +129,15 @@ runs:
         sudo swapon /mnt/ci-swapfile
         free -h
 
-    # The udev rule is what makes /dev/kvm writable for the runner's user. KVM on hosted runners
-    # is documented for public repositories; `test -w` is what says whether this one has it.
-    - name: Enable KVM
-      if: inputs.kvm == 'true'
-      shell: bash
-      run: |
-        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' |
-          sudo tee /etc/udev/rules.d/99-kvm4all.rules
-        sudo udevadm control --reload-rules
-        sudo udevadm trigger --name-match=kvm
-        # The host of a hosted runner is not chosen: some have no nested virtualization. The step is
-        # named so that rerun-no-kvm.yaml can tell this failure from a failed suite and run it again.
-        test -w /dev/kvm || { echo "::error::this runner has no usable /dev/kvm"; exit 1; }
-
     # `fallback` builds a derivation from source when its substitute cannot be downloaded (a NAR of 460 MB
     # that the cache served wrongly once, with `Stream error in the HTTP/2 framing layer`), so one dropped
     # connection costs minutes of compiling in that job and not the job. HTTP/2 stays at its default.
     - name: Install nix
-      if: inputs.install == 'true'
+      if: steps.kvm.outputs.state != 'missing' && inputs.install == 'true'
       uses: cachix/install-nix-action@v31
       with:
+        # off: the step above is the one that enables KVM, and checks that it is usable
+        enable_kvm: false
         extra_nix_config: |
           accept-flake-config = false
           fallback = true
@@ -89,7 +146,7 @@ runs:
     # The token is handed over only where writing is wanted; without it the action only reads. The cache
     # is public, so reading needs no token.
     - name: Cachix
-      if: inputs.cachix-cache != ''
+      if: steps.kvm.outputs.state != 'missing' && inputs.cachix-cache != ''
       uses: cachix/cachix-action@v16
       with:
         name: ${{ inputs.cachix-cache }}

.github/workflows/_vm-attempt.yaml

@@ -0,0 +1,105 @@
+# One attempt at a list of VM suites, one machine each. Called by _vm.yaml, which calls it up to three
+# times: a hosted runner may land on a host with no KVM, and the host cannot be chosen, only drawn again.
+#
+# With `defer-no-kvm` a leg that lands on such a host does not fail: it uploads the artifact
+# `kvm-missing-<run attempt>-<attempt>-<suite>` and succeeds without having run the suite, and the next
+# attempt reads the artifacts to find the suites it has to run. The last attempt does not defer: a suite
+# that found no KVM in every attempt fails.
+
+name: vm-attempt
+
+on:
+  workflow_call:
+    inputs:
+      suites:
+        description: The suites to run, as a JSON list (the first attempt); later attempts find theirs
+        type: string
+        default: "[]"
+      attempt:
+        description: 1, 2 or 3
+        type: number
+        required: true
+      slow:
+        description: The slow tier (the driver is run by hand, and its output kept), not the sandboxed `nix build`
+        type: boolean
+        default: false
+      defer-no-kvm:
+        type: boolean
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  # the suites that the attempt before found no KVM for
+  plan:
+    if: ${{ inputs.attempt > 1 }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    permissions:
+      actions: read
+    outputs:
+      suites: ${{ steps.plan.outputs.suites }}
+    steps:
+      - id: plan
+        env:
+          GH_TOKEN: ${{ github.token }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+          ATTEMPT: ${{ inputs.attempt }}
+        run: |
+          # GitHub expressions cannot subtract: the attempt before is computed here
+          PREFIX="kvm-missing-$RUN_ATTEMPT-$((ATTEMPT - 1))-"
+          suites=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" \
+            --jq '.artifacts[].name' |
+            jq -R --arg prefix "$PREFIX" 'select(startswith($prefix)) | ltrimstr($prefix)' |
+            jq -sc .)
+          echo "suites to run again: $suites"
+          echo "suites=$suites" >> "$GITHUB_OUTPUT"
+
+  suite:
+    needs: [plan]
+    if: ${{ !cancelled() && needs.plan.result != 'failure' && (inputs.attempt == 1 || needs.plan.outputs.suites != '[]') }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: ${{ inputs.slow && 60 || 45 }}
+    strategy:
+      fail-fast: false
+      max-parallel: 8
+      matrix:
+        suite: ${{ fromJson(inputs.attempt == 1 && inputs.suites || needs.plan.outputs.suites) }}
+    steps:
+      - uses: actions/checkout@v6
+      - id: nix
+        uses: ./.github/actions/nix
+        with:
+          kvm: "true"
+          defer-no-kvm: ${{ inputs.defer-no-kvm }}
+          cachix-cache: ${{ vars.CACHIX_CACHE }}
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - name: Remember that this host had no KVM
+        if: steps.nix.outputs.kvm == 'missing'
+        run: echo "${{ matrix.suite }}" > kvm-missing
+      - if: steps.nix.outputs.kvm == 'missing'
+        uses: actions/upload-artifact@v7
+        with:
+          name: kvm-missing-${{ github.run_attempt }}-${{ inputs.attempt }}-${{ matrix.suite }}
+          path: kvm-missing
+          retention-days: 1
+
+      # each suite in its own machine, in a sandboxed `nix build` like any other check
+      - if: steps.nix.outputs.kvm == 'ready' && !inputs.slow
+        run: nix build -L --no-link .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}
+
+      # the driver is run by hand, outside the sandbox, so that the screenshots and the journals of a
+      # failed run can be uploaded
+      - if: steps.nix.outputs.kvm == 'ready' && inputs.slow
+        run: nix build -L .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}.driver -o driver
+      - if: steps.nix.outputs.kvm == 'ready' && inputs.slow
+        run: mkdir -p out && cd out && ../driver/bin/nixos-test-driver -o .
+      - if: ${{ always() && steps.nix.outputs.kvm == 'ready' && inputs.slow }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: ci-vm-${{ matrix.suite }}-${{ github.sha }}
+          path: out/
+          if-no-files-found: ignore
+          retention-days: 7

.github/workflows/_vm.yaml

@@ -0,0 +1,66 @@
+# The KDE VM suites of a tier, with the draw of the host repeated for the ones that found no KVM.
+#
+# GitHub documents hardware acceleration on its Linux runners, and does not promise it of every host of
+# the pool (docs/dev/ci.md, "KVM"). A suite whose machine has no /dev/kvm is run again on a new machine,
+# in the attempt after, up to three in all. Everything is in this one run, so that it works on any
+# branch and any event, and the check that a ruleset requires is the result of this workflow call: green
+# when every suite ran and passed, red when a suite failed, or when no machine had KVM in three draws.
+#
+# The caller grants `actions: read`, which the planning of the later attempts needs.
+
+name: vm
+
+on:
+  workflow_call:
+    inputs:
+      suites:
+        description: The suites of the tier, as a JSON list
+        type: string
+        required: true
+      slow:
+        description: The slow tier
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  attempt-1:
+    permissions:
+      contents: read
+      actions: read
+    uses: ./.github/workflows/_vm-attempt.yaml
+    with:
+      suites: ${{ inputs.suites }}
+      attempt: 1
+      slow: ${{ inputs.slow }}
+      defer-no-kvm: true
+    secrets: inherit
+
+  attempt-2:
+    needs: [attempt-1]
+    if: ${{ !cancelled() && needs.attempt-1.result != 'skipped' }}
+    permissions:
+      contents: read
+      actions: read
+    uses: ./.github/workflows/_vm-attempt.yaml
+    with:
+      attempt: 2
+      slow: ${{ inputs.slow }}
+      defer-no-kvm: true
+    secrets: inherit
+
+  # the last draw: a host with no KVM fails the suite
+  attempt-3:
+    needs: [attempt-2]
+    if: ${{ !cancelled() && needs.attempt-2.result != 'skipped' }}
+    permissions:
+      contents: read
+      actions: read
+    uses: ./.github/workflows/_vm-attempt.yaml
+    with:
+      attempt: 3
+      slow: ${{ inputs.slow }}
+      defer-no-kvm: false
+    secrets: inherit

.github/workflows/ci.yaml

@@ -333,25 +333,18 @@ jobs:
           tiers=$(nix eval --json .#legacyPackages.x86_64-linux.vmTiers)
           echo "fast=$(jq -c '.fast // []' <<<"$tiers")" >> "$GITHUB_OUTPUT"
 
-  # each suite in its own machine, in a sandboxed `nix build` like any other check
+  # Each suite in its own machine, in a sandboxed `nix build` like any other check. A machine with no
+  # KVM is drawn again, three draws at most (_vm.yaml); `actions: read` is for the planning of the later draws.
   vm-fast:
     needs: [discover]
     if: ${{ !cancelled() && !failure() && needs.discover.result == 'success' && needs.discover.outputs.fast != '[]' }}
-    runs-on: ubuntu-24.04
-    timeout-minutes: 45
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix:
-        suite: ${{ fromJson(needs.discover.outputs.fast) }}
-    steps:
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/nix
-        with:
-          kvm: "true"
-          cachix-cache: ${{ vars.CACHIX_CACHE }}
-          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-      - run: nix build -L --no-link .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}
+    permissions:
+      contents: read
+      actions: read
+    uses: ./.github/workflows/_vm.yaml
+    with:
+      suites: ${{ needs.discover.outputs.fast }}
+    secrets: inherit
 
   # ---- stage 5: macOS ----
 

.github/workflows/rerun-no-kvm.yaml

@@ -1,38 +0,0 @@
-# A hosted runner of a repository may land on a host with no KVM, which fails the "Enable
-# KVM" step of the nix action (.github/actions/nix). Another attempt usually gets another host, so a run that failed
-# only for that is rerun (the API lists a composite action as one step, "Run ./.github/actions/nix", not
-# as the steps inside it, so that is the step that is looked for in the jobs that need KVM), up to three attempts. A suite that fails is left alone. A run cannot rerun
-# its own failed jobs while it is still in progress, hence a workflow of its own.
-
-name: rerun-no-kvm
-
-on:
-  workflow_run:
-    workflows: [ci, sweeps]
-    types: [completed]
-
-jobs:
-  rerun:
-    if: >-
-      github.repository == 'LGUG2Z/komorebi' &&
-      github.event.workflow_run.conclusion == 'failure' &&
-      github.event.workflow_run.run_attempt < 3
-    runs-on: ubuntu-24.04
-    permissions:
-      actions: write
-    steps:
-      - name: Rerun the failed jobs when every one of them failed to get KVM
-        env:
-          GH_TOKEN: ${{ github.token }}
-          RUN_ID: ${{ github.event.workflow_run.id }}
-          REPO: ${{ github.repository }}
-        run: |
-          # for each failed job, the name of the step that failed
-          failed=$(gh api --paginate "repos/$REPO/actions/runs/$RUN_ID/jobs?filter=latest" \
-            --jq '.jobs[] | select(.conclusion == "failure") | [.name, ([.steps[] | select(.conclusion == "failure") | .name] | first // "")] | @tsv')
-          echo "$failed"
-          if [ -z "$failed" ] || printf '%s\n' "$failed" | awk -F'\t' '!($1 ~ /^(vm-|canary)/ && $2 ~ /(Enable KVM|actions\/nix)/) { bad = 1 } END { exit !bad }'; then
-            echo "something other than a missing /dev/kvm failed: not rerunning"
-            exit 0
-          fi
-          gh run rerun "$RUN_ID" --repo "$REPO" --failed

.github/workflows/sweeps.yaml

@@ -46,33 +46,19 @@ jobs:
           echo "slow=$(jq -c '.slow // []' <<<"$tiers")" >> "$GITHUB_OUTPUT"
 
   # The suites that draw and move a pointer. The driver is run by hand, outside the sandbox, so that the
-  # screenshots and the journals of a failed run can be uploaded.
+  # screenshots and the journals of a failed run can be uploaded (_vm-attempt.yaml). A machine with no
+  # KVM is drawn again, three draws at most (_vm.yaml).
   vm-slow:
     needs: [discover]
     if: ${{ needs.discover.outputs.slow != '[]' }}
-    runs-on: ubuntu-24.04
-    timeout-minutes: 60
-    strategy:
-      fail-fast: false
-      max-parallel: 8
-      matrix:
-        suite: ${{ fromJson(needs.discover.outputs.slow) }}
-    steps:
-      - uses: actions/checkout@v6
-      - uses: ./.github/actions/nix
-        with:
-          kvm: "true"
-          cachix-cache: ${{ vars.CACHIX_CACHE }}
-          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
-      - run: nix build -L .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}.driver -o driver
-      - run: mkdir -p out && cd out && ../driver/bin/nixos-test-driver -o .
-      - if: always()
-        uses: actions/upload-artifact@v7
-        with:
-          name: ci-vm-${{ matrix.suite }}-${{ github.sha }}
-          path: out/
-          if-no-files-found: ignore
-          retention-days: 7
+    permissions:
+      contents: read
+      actions: read
+    uses: ./.github/workflows/_vm.yaml
+    with:
+      suites: ${{ needs.discover.outputs.slow }}
+      slow: true
+    secrets: inherit
 
   # Evaluating every NixOS system of the module and of the Plasma guests peaks at about 11.6 GB (see
   # docs/dev/flake-checks.md): the swap is for that, and the free disk for the guests' closures.