fix: run other k3s clusters as kubevirt quests, use the interface provided by kubevirt

README.md

@@ -24,8 +24,8 @@ This repository is home to the nix code that builds my systems:
 1. NixOS Desktops: NixOS with home-manager, i3, hyprland, agenix, etc.
 2. macOS Desktops: nix-darwin with home-manager, share the same home-manager configuration with
    NixOS Desktops.
-3. NixOS Servers: virtual machines running on Proxmox, with various services, such as kubernetes,
+3. NixOS Servers: virtual machines running on Proxmox/KubeVirt, with various services, such as
-   homepage, prometheus, grafana, etc.
+   kubernetes, homepage, prometheus, grafana, etc.
 
 See [./hosts](./hosts) for details of each host.
 

hosts/12kingdoms-rakushun/homepage/config/services.yaml

@@ -2,24 +2,31 @@
 # For configuration options and examples, please see:
 # https://gethomepage.dev/latest/configs/services
 
-- Proxmox VE 虚拟化集群:
+- KubeVirt 虚拟化集群:
-    - PVE-UM560:
+    - KubeVirt-Shoryu:
-        icon: si-proxmox
+        icon: si-kubevirt
-        href: https://192.168.5.173:8006/
-        description: "CPU: R5-5625U / MEM: 32G / DISK: 512G+4T*2"
-        siteMonitor: https://192.168.5.173:8006/
-
-    - PVE-S500Plus:
-        icon: si-proxmox
-        href: https://192.168.5.174:8006/
         description: "CPU: R7-5825U / MEM: 64G / DISK: 1T"
-        siteMonitor: https://192.168.5.174:8006/
+        href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.181:9100
-
+        siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.181:9100
-    - PVE-GTR5:
+    - KubeVirt-Shushou:
-        icon: si-proxmox
+        icon: si-kubevirt
-        href: https://192.168.5.172:8006/
         description: "CPU: R9-5900HX / MEM: 64G / DISK: 1T"
-        siteMonitor: https://192.168.5.172:8006/
+        href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.182:9100
+        siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.182:9100
+    - KubeVirt-Youko:
+        icon: si-kubevirt
+        description: "CPU: R5-5625U / MEM: 32G / DISK: 512G+4T*2"
+        href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.183:9100
+        siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.183:9100
+    - LongHorn-Storage:
+        icon: si-longhorn
+        href: http://longhorn.writefor.fun/
+    - Victoria-Metrics:
+        icon: si-victoriametrics
+        href: http://vm.writefor.fun/
+    - KubeVirt-Grafana:
+        icon: si-grafana
+        href: http://k8s-grafana.writefor.fun/
 
 - Homelab Monitoring:
     - Grafana:

hosts/12kingdoms-rakushun/homepage/config/settings.yaml

@@ -40,8 +40,8 @@ useEqualHeights: true
 # Groups and its layout
 # Groups Name should match the name defined in your services.yaml or widgets.yaml
 layout:
-  Proxmox VE 虚拟化集群:
+  KubeVirt 虚拟化集群:
-    icon: si-proxmox
+    icon: si-kubevirt
     tab: First
 
   Group A:

hosts/12kingdoms-rakushun/restic.nix

@@ -8,7 +8,7 @@ in {
     homelab-backup = {
       inherit passwordFile;
       initialize = true; # Initialize the repository if it doesn't exist.
-      repository = "rclone:smb-downloads:/Downloads/proxmox-backup/"; # backup to a rclone remote
+      repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote
 
       # rclone related
       # rcloneOptions = {
@@ -34,20 +34,16 @@ in {
       # A script that must run before starting the backup process.
       backupPrepareCommand = ''
         ${pkgs.nushell}/bin/nu -c '
-          let pve_nodes = [
+          let kubevirt_nodes = [
-            # proxmox cluster's nodes
+            "kubevirt-shoryu"
-            "um560"
+            "kubevirt-shushou"
-            "gtr5"
+            "kubevirt-youko"
-            "s500plus"
-
-            # others
-            "kana"
           ]
 
-          pve_nodes | each {|it|
+          kubevirt_nodes | each {|it|
             rsync -avz \
             -e "ssh -i ${sshKeyPath}"  \
-            $"($it):/var/lib/vz" $"/tmp/restic-backup-temp/($it)"
+            $"($it):/perissitent/" $"/tmp/restic-backup-temp/($it)"
           }
         '
       '';

hosts/idols-aquamarine/default.nix

@@ -1,7 +1,7 @@
 {myvars, ...}:
 #############################################################
 #
-#  Aquamarine - A NixOS VM running on Proxmox
+#  Aquamarine - A NixOS VM running on Proxmox/KubeVirt
 #
 #############################################################
 let

hosts/idols-kana/default.nix

@@ -5,7 +5,7 @@
 }:
 #############################################################
 #
-#  Kana - a NixOS VM running on Proxmox
+#  Kana - a NixOS VM running on Proxmox/KubeVirt
 #
 #############################################################
 let

hosts/idols-ruby/default.nix

@@ -5,7 +5,7 @@
 }:
 #############################################################
 #
-#  Ruby - a NixOS VM running on Proxmox
+#  Ruby - a NixOS VM running on Proxmox/KubeVirt
 #
 #############################################################
 let

hosts/k8s/k3s-prod-1-master-1/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-master-1"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-prod-1-master-2/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-master-2"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-prod-1-master-3/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-master-3"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-prod-1-worker-1/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-worker-1"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-prod-1-worker-2/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-worker-2"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-prod-1-worker-3/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-prod-1-worker-3"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-test-1-master-1/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-test-1-master-1"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-test-1-master-2/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-test-1-master-2"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/k3s-test-1-master-3/default.nix

@@ -7,7 +7,7 @@
 }: let
   hostName = "k3s-test-1-master-3"; # define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtGuestModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/kubevirt-shoryu/default.nix

@@ -9,7 +9,7 @@
   # MoreFine - S500Plus
   hostName = "kubevirt-shoryu"; # Define your hostname.
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtHostModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/kubevirt-shushou/default.nix

@@ -7,9 +7,8 @@
   ...
 }: let
   hostName = "kubevirt-shushou"; # Define your hostname.
-  k3sServerName = "kubevirt-shoryu";
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtHostModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

hosts/k8s/kubevirt-youko/default.nix

@@ -7,9 +7,8 @@
   ...
 }: let
   hostName = "kubevirt-youko"; # Define your hostname.
-  k3sServerName = "kubevirt-shoryu";
 
-  coreModule = mylib.genKubeVirtCoreModule {
+  coreModule = mylib.genKubeVirtHostModule {
     inherit pkgs hostName;
     inherit (myvars) networking;
   };

lib/default.nix

@@ -5,9 +5,10 @@
 
   attrs = import ./attrs.nix {inherit lib;};
 
-  genKubeVirtCoreModule = import ./genKubeVirtCoreModule.nix;
   genK3sServerModule = import ./genK3sServerModule.nix;
   genK3sAgentModule = import ./genK3sAgentModule.nix;
+  genKubeVirtHostModule = import ./genKubeVirtHostModule.nix;
+  genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix;
 
   # use path relative to the root of the project
   relativeToRoot = lib.path.append ../.;

lib/genKubeVirtGuestModule.nix

@@ -0,0 +1,33 @@
+{
+  pkgs,
+  hostName,
+  networking,
+  ...
+}: let
+  inherit (networking.hostsAddr.${hostName}) iface;
+in {
+  # supported file systems, so we can mount any removable disks with these filesystems
+  boot.supportedFilesystems = [
+    "ext4"
+    "btrfs"
+    "xfs"
+    "fat"
+    "vfat"
+    "exfat"
+  ];
+
+  networking = {
+    inherit hostName;
+    inherit (networking) defaultGateway nameservers;
+    inherit (networking.hostsInterface.${hostName}) interfaces;
+    networkmanager.enable = false;
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "23.11"; # Did you read the comment?
+}

lib/nixosSystem.nix

@@ -17,13 +17,6 @@ in
       nixos-modules
       ++ [
         nixos-generators.nixosModules.all-formats
-        {
-          # formatConfigs.iso = {config, ...}: {};
-          formatConfigs.proxmox = {config, ...}: {
-            # custom proxmox's image name
-            proxmox.qemuConf.name = "${config.networking.hostName}-nixos-${config.system.nixos.label}";
-          };
-        }
       ]
       ++ (
         lib.optionals ((lib.lists.length home-modules) > 0)

modules/nixos/server/kubevirt-hardware-configuration.nix

@@ -27,7 +27,8 @@
 
     services.qemuGuest.enable = true;
     services.openssh.enable = true;
-    services.cloud-init.enable = true;
+    # we configure the host via nixos itself, so we don't need the cloud-init
+    services.cloud-init.enable = false;
     systemd.services."serial-getty@ttyS0".enable = true;
   };
 }

modules/nixos/server/proxmox-hardware-configuration.nix

@@ -1,36 +0,0 @@
-{lib, ...}:
-##############################################################################
-#
-#  Template for Proxmox's VM, mainly based on:
-#    https://github.com/NixOS/nixpkgs/blob/nixos-unstable/nixos/modules/virtualisation/proxmox-image.nix
-#
-#  the url above is used by `nixos-generator` to generate the Proxmox's VMA image file.
-#
-##############################################################################
-{
-  boot = {
-    # after resize the disk, it will grow partition automatically.
-    growPartition = true;
-    kernelParams = ["console=ttyS0"];
-    loader.grub = {
-      device = "/dev/vda";
-
-      # we do not support EFI, so disable it.
-      efiSupport = false;
-      efiInstallAsRemovable = false;
-    };
-
-    loader.timeout = lib.mkForce 3; # wait for 3 seconds to select the boot entry
-    initrd.availableKernelModules = ["uas" "virtio_blk" "virtio_pci"];
-  };
-
-  fileSystems."/" = {
-    device = "/dev/disk/by-label/nixos";
-    autoResize = true;
-    fsType = "ext4";
-  };
-  # we do not have a /boot partition, so do not mount it.
-
-  # it alse had qemu-guest-agent installed by default.
-  services.qemuGuest.enable = lib.mkDefault true;
-}

outputs/x86_64-linux/src/idols-aquamarine.nix

@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/idols-kana.nix

@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/idols-ruby.nix

@@ -39,8 +39,7 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 
   # nixos tests
   packages."${name}-nixos-tests" = import ../nixos-tests/idols-ruby.nix systemArgs;

outputs/x86_64-linux/src/k3s-prod-1-master-1.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -39,6 +39,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-prod-1-master-2.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-prod-1-master-3.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-prod-1-worker-1.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-prod-1-worker-2.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-prod-1-worker-3.nix

@@ -20,7 +20,7 @@
         # common
         "secrets/nixos.nix"
         "modules/nixos/server/server.nix"
-        "modules/nixos/server/proxmox-hardware-configuration.nix"
+        "modules/nixos/server/kubevirt-hardware-configuration.nix"
         # host specific
         "hosts/k8s/${name}"
       ])
@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
+  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
-  packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
 }

outputs/x86_64-linux/src/k3s-test-1-master-1.nix

@@ -39,6 +39,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
   packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
 }

outputs/x86_64-linux/src/k3s-test-1-master-2.nix

@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
   packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
 }

outputs/x86_64-linux/src/k3s-test-1-master-3.nix

@@ -36,6 +36,5 @@ in {
   colmena.${name} =
     mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
 
-  # generate proxmox image for virtual machines without desktop environment
   packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
 }

vars/networking.nix

@@ -36,17 +36,17 @@
     };
     aquamarine = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.101";
     };
     ruby = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.102";
     };
     kana = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.103";
     };
     nozomi = {
@@ -61,7 +61,7 @@
     };
     chiaya = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.106";
     };
     suzu = {
@@ -90,32 +90,32 @@
     # ============================================
     k3s-prod-1-master-1 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.108";
     };
     k3s-prod-1-master-2 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.109";
     };
     k3s-prod-1-master-3 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.110";
     };
     k3s-prod-1-worker-1 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.111";
     };
     k3s-prod-1-worker-2 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.112";
     };
     k3s-prod-1-worker-3 = {
       # VM
-      iface = "ens18";
+      iface = "eth1";
       ipv4 = "192.168.5.113";
     };