From bd2b8594f7db6d909b5e0aa8f810d71a5f09c702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ryan4yin=20=7C=20=E4=BA=8C=E8=8A=B1?= Date: Tue, 17 Nov 2026 19:37:29 +0800 Subject: [PATCH] feat(nixos): reboot physical hosts on hard freeze or panic (#528) Assisted-by: DeepSeek Harness (claude-opus-5-5) --- docs/WORKAROUNDS.md | 15 ++++++++------- hosts/idols-aquamarine/default.nix | 1 + lib/genVmHostModule.nix | 5 ++++- modules/nixos/desktop/default.nix | 5 ++++- modules/nixos/watchdog.nix | 6 ++++++ 5 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 modules/nixos/watchdog.nix diff --git a/docs/WORKAROUNDS.md b/docs/WORKAROUNDS.md index 39309aa2..f1edba99 100644 --- a/docs/WORKAROUNDS.md +++ b/docs/WORKAROUNDS.md @@ -66,13 +66,14 @@ only delete inactive duplicates. A known limitation with no fix yet. -| ID | What & where | Why | Removal condition | Added | Revisit | Status | -| ------ | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | -| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | -| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | -| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | -| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active | -| WA-035 | SSH retry + multiplexing to LAN peers, ARP fallback tuning — `home/base/tui/ssh.nix`, `hosts/idols-ai/neighbour-probe.nix` | the desktop reaches the LAN over a Mesh WiFi backhaul, which drops unicast to a peer whose MAC it has aged out of its forwarding table while still flooding broadcast: a capture shows seven SYNs from one socket addressed to the cached MAC with no answer, three unicast ARP probes unanswered, then a broadcast ARP answered in 0.4 ms after ~12 s of failed connects. The ssh settings retry a short connect and keep a master warm so the entry stays fresh; the neighbour-probe tunings walk the kernel to that broadcast fallback in ~1 s | Drop once the backhaul is wired or the mesh stops dropping unknown unicast | 2026-10-16 | when the backhaul changes | active | +| ID | What & where | Why | Removal condition | Added | Revisit | Status | +| ------ | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------------------------------------------------------------- | ------ | +| WA-016 | subscription rule snapshots — `modules/nixos/desktop/networking/mihomo/` (`rules_from:`) | mihomo's `proxy-providers` imports proxies only, so the `rules` and `proxy-groups` a subscription ships with are dropped by the core; the generator re-imports them from a local YAML copy that has to be refreshed by hand when the provider changes its routing policy | Drop the snapshots once mihomo can import rules from a proxy-provider, or the providers publish their rules as rule-providers | 2026-10-04 | when mihomo gains rule import | active | +| WA-022 | qemu-user binfmt removed — `hosts/12kingdoms-shoukei/hardware-configuration.nix` | the x86-on-arm module owns the x86 ELF handlers and this host does not need riscv64, but dropping `boot.binfmt.emulatedSystems` also drops `nix.settings.extra-platforms` and qemu-user, so `podman run --platform=linux/amd64`, x86 chroots and local builds of uncached x86 derivations need a remote x86_64 builder | Restore `boot.binfmt.emulatedSystems` (without x86_64/i686, which the module asserts against) if such a workload is needed | 2026-10-07 | when an amd64 container/chroot or uncached x86 build is needed | active | +| WA-025 | brcmfmac 4-way-handshake offload authentication — upstream kernel/wpa_supplicant, no code in this repo | Since hostap commit 416386060, wpa_supplicant 2.11+ expects `NL80211_CMD_PORT_AUTHORIZED` from drivers that offload the 4-way handshake; brcmfmac never reports it, so authentication intermittently fails on Apple Silicon (BRCM4378/4387/4388). nixpkgs already carries the Broadcom revert in wpa_supplicant 2.12; the proper fix, Janne Grunau's `wifi: brcmfmac: cfg80211: Report port_authorized for 4-way HS offload` (2026-09), is still under review. Entering the key again after such a failure only creates a duplicate profile, so cancel and retry instead | Fix lands via a nixpkgs kernel update (or by carrying the patch through `nixpkgs-patched`) | 2026-10-07 | on every kernel bump | active | +| WA-033 | USB-C/XHCI port `_PLD` ACPI noise — this board's DSDT, no code in this repo | every boot logs 16 `ACPI Error: Aborting method \_SB.PC00.TXHC.RHUB.SS01._PLD ... (AE_NOT_FOUND)` lines, plus `\_SB.PC02.XHCI.RHUB.HS01._PLD`; the ports work, and these messages are unrelated to the GPU's ATPX failure, which names `\_SB.PC00.GFX0.ATPX` instead | Drop if a BIOS update defines the `_PLD` descriptors | 2026-10-14 | on a BIOS update | active | +| WA-035 | SSH retry + multiplexing to LAN peers, ARP fallback tuning — `home/base/tui/ssh.nix`, `hosts/idols-ai/neighbour-probe.nix` | the desktop reaches the LAN over a Mesh WiFi backhaul, which drops unicast to a peer whose MAC it has aged out of its forwarding table while still flooding broadcast: a capture shows seven SYNs from one socket addressed to the cached MAC with no answer, three unicast ARP probes unanswered, then a broadcast ARP answered in 0.4 ms after ~12 s of failed connects. The ssh settings retry a short connect and keep a master warm so the entry stays fresh; the neighbour-probe tunings walk the kernel to that broadcast fallback in ~1 s | Drop once the backhaul is wired or the mesh stops dropping unknown unicast | 2026-10-16 | when the backhaul changes | active | +| WA-052 | Hardware watchdog armed through suspend — `modules/nixos/watchdog.nix` | `RuntimeWatchdogSec` arms the board watchdog on every physical host; it is not yet verified that `iTCO_wdt` (aquamarine, ai) and `apple_wdt` (shoukei) pause it while suspended. A watchdog that keeps counting would reset the host during sleep (aquamarine idle suspend, shoukei lid suspend) | Drop once aquamarine, ai and shoukei each resume from a suspend longer than 30 s without a reset; if one resets, disable the runtime watchdog on that host | 2026-11-17 | after the first suspend on each of those hosts | active | ## WIP diff --git a/hosts/idols-aquamarine/default.nix b/hosts/idols-aquamarine/default.nix index 40a324f2..41bf1710 100644 --- a/hosts/idols-aquamarine/default.nix +++ b/hosts/idols-aquamarine/default.nix @@ -20,6 +20,7 @@ in imports = [ disko.nixosModules.default ../../modules/nixos/power.nix + ../../modules/nixos/watchdog.nix # disks ./disko-fs.nix diff --git a/lib/genVmHostModule.nix b/lib/genVmHostModule.nix index b650d955..a98511de 100644 --- a/lib/genVmHostModule.nix +++ b/lib/genVmHostModule.nix @@ -11,7 +11,10 @@ let in { # CPU policy belongs to physical hosts; guests do not control hardware frequency. - imports = [ ../modules/nixos/power.nix ]; + imports = [ + ../modules/nixos/power.nix + ../modules/nixos/watchdog.nix + ]; # supported file systems, so we can mount any removable disks with these filesystems boot.supportedFilesystems = [ diff --git a/modules/nixos/desktop/default.nix b/modules/nixos/desktop/default.nix index e4906b58..3e1f9f55 100644 --- a/modules/nixos/desktop/default.nix +++ b/modules/nixos/desktop/default.nix @@ -1,6 +1,9 @@ { mylib, ... }: { - imports = mylib.scanPaths ./. ++ [ ../power.nix ]; + imports = mylib.scanPaths ./. ++ [ + ../power.nix + ../watchdog.nix + ]; # Power-supply reporting for the desktop shell: the laptop's battery, and # peripheral batteries on a desktop. UPower only reports state; unlike TuneD it diff --git a/modules/nixos/watchdog.nix b/modules/nixos/watchdog.nix new file mode 100644 index 00000000..dc750630 --- /dev/null +++ b/modules/nixos/watchdog.nix @@ -0,0 +1,6 @@ +{ + # Reboot physical hosts after a hard freeze or kernel panic instead of waiting for a power cycle; + # a host without a hardware watchdog only logs that /dev/watchdog is missing. + systemd.settings.Manager.RuntimeWatchdogSec = "30s"; + boot.kernel.sysctl."kernel.panic" = 20; # seconds; the default 0 hangs forever +}