feat(aquamarine): suspend after idle & wake-on-lan (#503)

Justfile

@@ -246,6 +246,12 @@ youko mode="switch":
 aquamarine mode="switch":
   colmena apply {{mode}} --build-on-target --on '@aquamarine' --verbose --show-trace
 
+# Wake every homelab host that has Wake-on-LAN armed.
+[linux]
+[group('homelab')]
+wake-homelab:
+  python3 {{absolute_path("scripts/wake-homelab.py")}}
+
 ############################################################################
 #
 # Commands for other Virtual Machines

hosts/idols-aquamarine/README.md

@@ -289,3 +289,40 @@ just local               # on aquamarine itself
 The install only touches the target disk; the staging host keeps booting from its own. To undo, move
 the disk back, restore the `hosts/idols-ai/disko-fs-data.nix` module this change deleted, and
 re-enroll that machine's TPM2 slot — see `../idols-ai/README.md`.
+
+## Wake-on-LAN
+
+BIOS: `Advanced > Power Setting` must have `Wake By Lan` enabled, and EUP must not cut standby power
+to the NIC. NixOS arms the RTL8125 through `systemd.network.links` using `OriginalName=en*`.
+
+After a boot, verify the link reports `Wake-on: g`:
+
+```bash
+ethtool enp133s0 | grep -E 'Supports Wake-on|Wake-on'
+```
+
+Wake every homelab host with Wake-on-LAN armed from another L2 host:
+
+```bash
+just wake-homelab
+```
+
+The command currently wakes only aquamarine; add a host to `scripts/wake-homelab.py` after its WoL
+link config and BIOS setting are in place. Test from a powered-off state, not from a reboot.
+
+## Idle suspend
+
+`autosuspend` checks every minute and suspends into S3 after 60 minutes without an active user
+session, meaningful CPU load, or an established connection to one of the request-serving ports:
+`18080` (ASR), `63081` (llama-swap/Strata), `63083`/`63084` (MagPie), and `65002` (direct Strata).
+It calls `systemctl suspend`, so systemd sleep inhibitors are also honored. WoL is armed by the
+`systemd.network.links` configuration above; no separate WoL service is needed.
+
+The daemon resets its idle timer after resume, so a wake does not immediately go back to sleep. To
+keep the host awake during maintenance:
+
+```bash
+systemd-inhibit --what=sleep --mode=block --who=maintenance --why="manual work" sleep infinity
+```
+
+Cancel it with Ctrl-C when finished.

hosts/idols-aquamarine/default.nix

@@ -36,6 +36,7 @@ in
 
     # services
     ./asr.nix
+    ./idle-suspend.nix
   ];
 
   # Bind the board's NCT6798D SuperIO, otherwise fan RPM and board temperatures
@@ -114,6 +115,13 @@ in
 
   # Predictable names follow the PCIe topology, so match the `en*` prefix instead of
   # a name; one wired port, so one link.
+  # `Name=` is not a valid [Match] key in systemd.link; systemd 262 ignores
+  # the whole file with "Unknown key 'Name'". Match the kernel name instead.
+  systemd.network.links."40-wake-on-lan" = {
+    matchConfig.OriginalName = "en*";
+    linkConfig.WakeOnLan = "magic";
+  };
+
   systemd.network.networks."10-lan" = {
     matchConfig.Name = [ "en*" ];
     networkConfig = {

hosts/idols-aquamarine/idle-suspend.nix

@@ -0,0 +1,55 @@
+{
+  myvars,
+  pkgs,
+  ...
+}:
+let
+  idleMinutes = 60;
+in
+{
+  services.autosuspend = {
+    enable = true;
+    # nixpkgs' pytest hook misses docutils for test_ext_autosuspend.py.
+    package =
+      (pkgs.autosuspend.override {
+        withDocs = false;
+        withMan = false;
+      }).overridePythonAttrs
+        (old: {
+          nativeCheckInputs =
+            (old.nativeCheckInputs or [ ])
+            ++ (with pkgs.python3Packages; [
+              sphinx
+              docutils
+            ]);
+        });
+    settings = {
+      interval = 60;
+      idle_time = idleMinutes * 60;
+      suspend_cmd = "${pkgs.systemd}/bin/systemctl suspend";
+    };
+    checks = {
+      Sessions = {
+        class = "LogindSessionsIdle";
+      };
+      Load = {
+        threshold = 1.0;
+      };
+      # ASR listens on 18080.
+      Asr = {
+        class = "ActiveConnection";
+        ports = toString myvars.ports.aquamarineAsr;
+      };
+      # llama-swap/Strata: 63081 is the proxy, 65002 the direct backend.
+      Inference = {
+        class = "ActiveConnection";
+        ports = "63081,65002";
+      };
+      # MagPie API and web UI: 63083 and 63084.
+      MagPie = {
+        class = "ActiveConnection";
+        ports = "63083,63084";
+      };
+    };
+  };
+}

scripts/wake-homelab.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Wake every homelab host whose NixOS config arms Wake-on-LAN."""
+
+from __future__ import annotations
+
+import socket
+import sys
+
+# Keep this list limited to hosts that actually set Wake-on: g in NixOS.
+# Add a host here only after its WoL service and BIOS setting are in place.
+HOSTS = {
+    "aquamarine": "8c:32:23:36:81:84",
+}
+BROADCASTS = ("192.168.5.255", "255.255.255.255")
+PORT = 9
+PACKETS = 3
+
+
+def magic_packet(mac: str) -> bytes:
+    octets = bytes.fromhex(mac.replace(":", ""))
+    if len(octets) != 6:
+        raise ValueError(f"invalid MAC: {mac}")
+    return b"\xff" * 6 + octets * 16
+
+
+def main() -> int:
+    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+    failed = False
+
+    for host, mac in HOSTS.items():
+        packet = magic_packet(mac)
+        print(f"waking {host} ({mac})")
+        for _ in range(PACKETS):
+            for broadcast in BROADCASTS:
+                try:
+                    sock.sendto(packet, (broadcast, PORT))
+                except OSError as exc:
+                    print(f"  {broadcast}:{PORT}: {exc}", file=sys.stderr)
+                    failed = True
+
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())