From 996330f19b657752e9c4451aeaef742c03a906a1 Mon Sep 17 00:00:00 2001 From: LGUG2Z Date: Fri, 6 Nov 2026 09:17:56 -0800 Subject: [PATCH] fix(ci): draw the host of a VM suite again when it has no KVM, in the same run A hosted runner of the ubuntu-24.04 pool is not promised nested virtualization, and 3 of the 16 suites of run 42182554717 failed the check of /dev/kvm. rerun-no-kvm.yaml could not help: a workflow_run workflow runs only from the default branch, where the file is not, and a run cannot rerun its own failed jobs while it is in progress. The step "Enable KVM" of the nix action is now the only one that enables KVM (enable_kvm of install-nix-action is off), runs before the disk and the install, waits for udev to settle and polls /dev/kvm for 30 seconds instead of testing it once, 42 ms after an asynchronous udevadm trigger, and prints what tells a missing device from a rule that was not applied yet. With defer-no-kvm a host without KVM is an output, not a failure. _vm.yaml calls _vm-attempt.yaml up to three times: the suites that found no KVM are listed through the artifacts of the run and run again on new machines, and the last attempt fails them. vm-fast and vm-slow are calls of it. rerun-no-kvm.yaml is removed. --- .github/actions/nix/action.yaml | 93 +++++++++++++++++++----- .github/workflows/_vm-attempt.yaml | 105 ++++++++++++++++++++++++++++ .github/workflows/_vm.yaml | 66 +++++++++++++++++ .github/workflows/ci.yaml | 25 +++---- .github/workflows/rerun-no-kvm.yaml | 38 ---------- .github/workflows/sweeps.yaml | 34 +++------ 6 files changed, 265 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/_vm-attempt.yaml create mode 100644 .github/workflows/_vm.yaml delete mode 100644 .github/workflows/rerun-no-kvm.yaml diff --git a/.github/actions/nix/action.yaml b/.github/actions/nix/action.yaml index 102eabaf..95a06fcd 100644 --- a/.github/actions/nix/action.yaml +++ b/.github/actions/nix/action.yaml @@ -21,6 +21,9 @@ inputs: kvm: description: Make /dev/kvm usable and tell nix that this machine can run NixOS tests default: "false" + defer-no-kvm: + description: "With kvm, a host without a usable /dev/kvm does not fail the step: the output kvm is `missing`, every other step of the action is skipped, and the caller runs the job again on another host (.github/workflows/_vm.yaml). Off, that host fails the step" + default: "false" free-disk: description: Delete the toolchains of the hosted image that this repository does not use (about 25 GB) default: "false" @@ -37,20 +40,86 @@ inputs: description: The token that may push to the cache, secrets.CACHIX_AUTH_TOKEN default: "" +outputs: + kvm: + description: "`ready` (kvm asked for and /dev/kvm is usable), `missing` (asked for with defer-no-kvm, and the host has none), empty (not asked for)" + value: ${{ steps.kvm.outputs.state }} + runs: using: composite steps: + # The udev rule is what makes /dev/kvm writable for the runner's user (the one GitHub documents: + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ ). + # `udevadm trigger` returns before the rule is applied, so the device is polled, for 30 seconds at + # most, and not tested once. The step comes first: a host without KVM is known in a second and not + # after the disk and the nix install. cachix/install-nix-action writes the same rule and ignores the + # outcome (`KVM is not available`), so its enable_kvm is off below: this step is the only one. + # + # The caller can tell the failure of this step from a failed suite: it is the step "Run + # ./.github/actions/nix" of the job (the API lists a composite action as one step), and with + # defer-no-kvm it is not a failure at all but the output kvm=missing. + - name: Enable KVM + id: kvm + if: inputs.kvm == 'true' + shell: bash + env: + DEFER: ${{ inputs.defer-no-kvm }} + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | + sudo tee /etc/udev/rules.d/99-kvm4all.rules + diagnose() { + echo "::group::KVM diagnostics ($1)" + ls -l /dev/kvm 2>&1 || true + id + getent group kvm || true + systemd-detect-virt || true + echo "cpu flags vmx/svm: $(grep -E -c 'vmx|svm' /proc/cpuinfo || true)" + lsmod | grep -E '^kvm' || echo "no kvm module listed" + echo "::endgroup::" + } + diagnose "before the rule" + # a module that is not loaded is the one case that the rule cannot fix; harmless where it is + sudo modprobe -q kvm_intel 2>/dev/null || sudo modprobe -q kvm_amd 2>/dev/null || true + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + sudo udevadm settle --timeout=30 || true + ready=false + for second in $(seq 0 30); do + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + echo "/dev/kvm is usable after $second s" + ready=true + break + fi + sleep 1 + done + if [ "$ready" = false ] && [ -e /dev/kvm ]; then + # the device is there and the rule did not reach it: set the mode by hand + echo "/dev/kvm exists and is not usable after 30 s: udev did not apply the rule, setting the mode by hand" + sudo chmod 0666 /dev/kvm && [ -r /dev/kvm ] && [ -w /dev/kvm ] && ready=true + fi + diagnose "after" + if [ "$ready" = true ]; then + echo "state=ready" >> "$GITHUB_OUTPUT" + elif [ "$DEFER" = true ]; then + echo "state=missing" >> "$GITHUB_OUTPUT" + echo "::warning::this host has no usable /dev/kvm ($([ -e /dev/kvm ] || echo 'the device is absent: no nested virtualization'))" + else + # the host of a hosted runner is not chosen: some have no nested virtualization + echo "::error::this runner has no usable /dev/kvm" + exit 1 + fi + # A Plasma guest is a closure of about 10 GB, and a hosted runner has little free space on / # until the toolchains it ships with are gone. - name: Free disk space - if: inputs.free-disk == 'true' || inputs.kvm == 'true' + if: steps.kvm.outputs.state != 'missing' && (inputs.free-disk == 'true' || inputs.kvm == 'true') shell: bash run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL df -h / - name: Add swap - if: inputs.swap == 'true' + if: steps.kvm.outputs.state != 'missing' && inputs.swap == 'true' shell: bash run: | sudo swapoff -a || true @@ -60,27 +129,15 @@ runs: sudo swapon /mnt/ci-swapfile free -h - # The udev rule is what makes /dev/kvm writable for the runner's user. KVM on hosted runners - # is documented for public repositories; `test -w` is what says whether this one has it. - - name: Enable KVM - if: inputs.kvm == 'true' - shell: bash - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | - sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - # The host of a hosted runner is not chosen: some have no nested virtualization. The step is - # named so that rerun-no-kvm.yaml can tell this failure from a failed suite and run it again. - test -w /dev/kvm || { echo "::error::this runner has no usable /dev/kvm"; exit 1; } - # `fallback` builds a derivation from source when its substitute cannot be downloaded (a NAR of 460 MB # that the cache served wrongly once, with `Stream error in the HTTP/2 framing layer`), so one dropped # connection costs minutes of compiling in that job and not the job. HTTP/2 stays at its default. - name: Install nix - if: inputs.install == 'true' + if: steps.kvm.outputs.state != 'missing' && inputs.install == 'true' uses: cachix/install-nix-action@v31 with: + # off: the step above is the one that enables KVM, and checks that it is usable + enable_kvm: false extra_nix_config: | accept-flake-config = false fallback = true @@ -89,7 +146,7 @@ runs: # The token is handed over only where writing is wanted; without it the action only reads. The cache # is public, so reading needs no token. - name: Cachix - if: inputs.cachix-cache != '' + if: steps.kvm.outputs.state != 'missing' && inputs.cachix-cache != '' uses: cachix/cachix-action@v16 with: name: ${{ inputs.cachix-cache }} diff --git a/.github/workflows/_vm-attempt.yaml b/.github/workflows/_vm-attempt.yaml new file mode 100644 index 00000000..0f5e8d8d --- /dev/null +++ b/.github/workflows/_vm-attempt.yaml @@ -0,0 +1,105 @@ +# One attempt at a list of VM suites, one machine each. Called by _vm.yaml, which calls it up to three +# times: a hosted runner may land on a host with no KVM, and the host cannot be chosen, only drawn again. +# +# With `defer-no-kvm` a leg that lands on such a host does not fail: it uploads the artifact +# `kvm-missing---` and succeeds without having run the suite, and the next +# attempt reads the artifacts to find the suites it has to run. The last attempt does not defer: a suite +# that found no KVM in every attempt fails. + +name: vm-attempt + +on: + workflow_call: + inputs: + suites: + description: The suites to run, as a JSON list (the first attempt); later attempts find theirs + type: string + default: "[]" + attempt: + description: 1, 2 or 3 + type: number + required: true + slow: + description: The slow tier (the driver is run by hand, and its output kept), not the sandboxed `nix build` + type: boolean + default: false + defer-no-kvm: + type: boolean + required: true + +permissions: + contents: read + +jobs: + # the suites that the attempt before found no KVM for + plan: + if: ${{ inputs.attempt > 1 }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read + outputs: + suites: ${{ steps.plan.outputs.suites }} + steps: + - id: plan + env: + GH_TOKEN: ${{ github.token }} + RUN_ATTEMPT: ${{ github.run_attempt }} + ATTEMPT: ${{ inputs.attempt }} + run: | + # GitHub expressions cannot subtract: the attempt before is computed here + PREFIX="kvm-missing-$RUN_ATTEMPT-$((ATTEMPT - 1))-" + suites=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts" \ + --jq '.artifacts[].name' | + jq -R --arg prefix "$PREFIX" 'select(startswith($prefix)) | ltrimstr($prefix)' | + jq -sc .) + echo "suites to run again: $suites" + echo "suites=$suites" >> "$GITHUB_OUTPUT" + + suite: + needs: [plan] + if: ${{ !cancelled() && needs.plan.result != 'failure' && (inputs.attempt == 1 || needs.plan.outputs.suites != '[]') }} + runs-on: ubuntu-24.04 + timeout-minutes: ${{ inputs.slow && 60 || 45 }} + strategy: + fail-fast: false + max-parallel: 8 + matrix: + suite: ${{ fromJson(inputs.attempt == 1 && inputs.suites || needs.plan.outputs.suites) }} + steps: + - uses: actions/checkout@v6 + - id: nix + uses: ./.github/actions/nix + with: + kvm: "true" + defer-no-kvm: ${{ inputs.defer-no-kvm }} + cachix-cache: ${{ vars.CACHIX_CACHE }} + cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Remember that this host had no KVM + if: steps.nix.outputs.kvm == 'missing' + run: echo "${{ matrix.suite }}" > kvm-missing + - if: steps.nix.outputs.kvm == 'missing' + uses: actions/upload-artifact@v7 + with: + name: kvm-missing-${{ github.run_attempt }}-${{ inputs.attempt }}-${{ matrix.suite }} + path: kvm-missing + retention-days: 1 + + # each suite in its own machine, in a sandboxed `nix build` like any other check + - if: steps.nix.outputs.kvm == 'ready' && !inputs.slow + run: nix build -L --no-link .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }} + + # the driver is run by hand, outside the sandbox, so that the screenshots and the journals of a + # failed run can be uploaded + - if: steps.nix.outputs.kvm == 'ready' && inputs.slow + run: nix build -L .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}.driver -o driver + - if: steps.nix.outputs.kvm == 'ready' && inputs.slow + run: mkdir -p out && cd out && ../driver/bin/nixos-test-driver -o . + - if: ${{ always() && steps.nix.outputs.kvm == 'ready' && inputs.slow }} + uses: actions/upload-artifact@v7 + with: + name: ci-vm-${{ matrix.suite }}-${{ github.sha }} + path: out/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/_vm.yaml b/.github/workflows/_vm.yaml new file mode 100644 index 00000000..c8f25798 --- /dev/null +++ b/.github/workflows/_vm.yaml @@ -0,0 +1,66 @@ +# The KDE VM suites of a tier, with the draw of the host repeated for the ones that found no KVM. +# +# GitHub documents hardware acceleration on its Linux runners, and does not promise it of every host of +# the pool (docs/dev/ci.md, "KVM"). A suite whose machine has no /dev/kvm is run again on a new machine, +# in the attempt after, up to three in all. Everything is in this one run, so that it works on any +# branch and any event, and the check that a ruleset requires is the result of this workflow call: green +# when every suite ran and passed, red when a suite failed, or when no machine had KVM in three draws. +# +# The caller grants `actions: read`, which the planning of the later attempts needs. + +name: vm + +on: + workflow_call: + inputs: + suites: + description: The suites of the tier, as a JSON list + type: string + required: true + slow: + description: The slow tier + type: boolean + default: false + +permissions: + contents: read + +jobs: + attempt-1: + permissions: + contents: read + actions: read + uses: ./.github/workflows/_vm-attempt.yaml + with: + suites: ${{ inputs.suites }} + attempt: 1 + slow: ${{ inputs.slow }} + defer-no-kvm: true + secrets: inherit + + attempt-2: + needs: [attempt-1] + if: ${{ !cancelled() && needs.attempt-1.result != 'skipped' }} + permissions: + contents: read + actions: read + uses: ./.github/workflows/_vm-attempt.yaml + with: + attempt: 2 + slow: ${{ inputs.slow }} + defer-no-kvm: true + secrets: inherit + + # the last draw: a host with no KVM fails the suite + attempt-3: + needs: [attempt-2] + if: ${{ !cancelled() && needs.attempt-2.result != 'skipped' }} + permissions: + contents: read + actions: read + uses: ./.github/workflows/_vm-attempt.yaml + with: + attempt: 3 + slow: ${{ inputs.slow }} + defer-no-kvm: false + secrets: inherit diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 44d62326..791767b9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -333,25 +333,18 @@ jobs: tiers=$(nix eval --json .#legacyPackages.x86_64-linux.vmTiers) echo "fast=$(jq -c '.fast // []' <<<"$tiers")" >> "$GITHUB_OUTPUT" - # each suite in its own machine, in a sandboxed `nix build` like any other check + # Each suite in its own machine, in a sandboxed `nix build` like any other check. A machine with no + # KVM is drawn again, three draws at most (_vm.yaml); `actions: read` is for the planning of the later draws. vm-fast: needs: [discover] if: ${{ !cancelled() && !failure() && needs.discover.result == 'success' && needs.discover.outputs.fast != '[]' }} - runs-on: ubuntu-24.04 - timeout-minutes: 45 - strategy: - fail-fast: false - max-parallel: 8 - matrix: - suite: ${{ fromJson(needs.discover.outputs.fast) }} - steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/nix - with: - kvm: "true" - cachix-cache: ${{ vars.CACHIX_CACHE }} - cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - - run: nix build -L --no-link .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }} + permissions: + contents: read + actions: read + uses: ./.github/workflows/_vm.yaml + with: + suites: ${{ needs.discover.outputs.fast }} + secrets: inherit # ---- stage 5: macOS ---- diff --git a/.github/workflows/rerun-no-kvm.yaml b/.github/workflows/rerun-no-kvm.yaml deleted file mode 100644 index 30ce39bd..00000000 --- a/.github/workflows/rerun-no-kvm.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# A hosted runner of a repository may land on a host with no KVM, which fails the "Enable -# KVM" step of the nix action (.github/actions/nix). Another attempt usually gets another host, so a run that failed -# only for that is rerun (the API lists a composite action as one step, "Run ./.github/actions/nix", not -# as the steps inside it, so that is the step that is looked for in the jobs that need KVM), up to three attempts. A suite that fails is left alone. A run cannot rerun -# its own failed jobs while it is still in progress, hence a workflow of its own. - -name: rerun-no-kvm - -on: - workflow_run: - workflows: [ci, sweeps] - types: [completed] - -jobs: - rerun: - if: >- - github.repository == 'LGUG2Z/komorebi' && - github.event.workflow_run.conclusion == 'failure' && - github.event.workflow_run.run_attempt < 3 - runs-on: ubuntu-24.04 - permissions: - actions: write - steps: - - name: Rerun the failed jobs when every one of them failed to get KVM - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ github.event.workflow_run.id }} - REPO: ${{ github.repository }} - run: | - # for each failed job, the name of the step that failed - failed=$(gh api --paginate "repos/$REPO/actions/runs/$RUN_ID/jobs?filter=latest" \ - --jq '.jobs[] | select(.conclusion == "failure") | [.name, ([.steps[] | select(.conclusion == "failure") | .name] | first // "")] | @tsv') - echo "$failed" - if [ -z "$failed" ] || printf '%s\n' "$failed" | awk -F'\t' '!($1 ~ /^(vm-|canary)/ && $2 ~ /(Enable KVM|actions\/nix)/) { bad = 1 } END { exit !bad }'; then - echo "something other than a missing /dev/kvm failed: not rerunning" - exit 0 - fi - gh run rerun "$RUN_ID" --repo "$REPO" --failed diff --git a/.github/workflows/sweeps.yaml b/.github/workflows/sweeps.yaml index a09010bc..d9b57c1c 100644 --- a/.github/workflows/sweeps.yaml +++ b/.github/workflows/sweeps.yaml @@ -46,33 +46,19 @@ jobs: echo "slow=$(jq -c '.slow // []' <<<"$tiers")" >> "$GITHUB_OUTPUT" # The suites that draw and move a pointer. The driver is run by hand, outside the sandbox, so that the - # screenshots and the journals of a failed run can be uploaded. + # screenshots and the journals of a failed run can be uploaded (_vm-attempt.yaml). A machine with no + # KVM is drawn again, three draws at most (_vm.yaml). vm-slow: needs: [discover] if: ${{ needs.discover.outputs.slow != '[]' }} - runs-on: ubuntu-24.04 - timeout-minutes: 60 - strategy: - fail-fast: false - max-parallel: 8 - matrix: - suite: ${{ fromJson(needs.discover.outputs.slow) }} - steps: - - uses: actions/checkout@v6 - - uses: ./.github/actions/nix - with: - kvm: "true" - cachix-cache: ${{ vars.CACHIX_CACHE }} - cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - - run: nix build -L .#legacyPackages.x86_64-linux.vm.${{ matrix.suite }}.driver -o driver - - run: mkdir -p out && cd out && ../driver/bin/nixos-test-driver -o . - - if: always() - uses: actions/upload-artifact@v7 - with: - name: ci-vm-${{ matrix.suite }}-${{ github.sha }} - path: out/ - if-no-files-found: ignore - retention-days: 7 + permissions: + contents: read + actions: read + uses: ./.github/workflows/_vm.yaml + with: + suites: ${{ needs.discover.outputs.slow }} + slow: true + secrets: inherit # Evaluating every NixOS system of the module and of the Plasma guests peaks at about 11.6 GB (see # docs/dev/flake-checks.md): the swap is for that, and the free disk for the guests' closures.