fix: run other k3s clusters as kubevirt quests, use the interface provided by kubevirt

This commit is contained in:
Ryan Yin
2024-03-31 23:58:38 +08:00
parent 7486e75016
commit c628eaa6c8
38 changed files with 111 additions and 130 deletions

View File

@@ -24,8 +24,8 @@ This repository is home to the nix code that builds my systems:
1. NixOS Desktops: NixOS with home-manager, i3, hyprland, agenix, etc.
2. macOS Desktops: nix-darwin with home-manager, share the same home-manager configuration with
NixOS Desktops.
3. NixOS Servers: virtual machines running on Proxmox, with various services, such as kubernetes,
homepage, prometheus, grafana, etc.
3. NixOS Servers: virtual machines running on Proxmox/KubeVirt, with various services, such as
kubernetes, homepage, prometheus, grafana, etc.
See [./hosts](./hosts) for details of each host.

View File

@@ -2,24 +2,31 @@
# For configuration options and examples, please see:
# https://gethomepage.dev/latest/configs/services
- Proxmox VE 虚拟化集群:
- PVE-UM560:
icon: si-proxmox
href: https://192.168.5.173:8006/
description: "CPU: R5-5625U / MEM: 32G / DISK: 512G+4T*2"
siteMonitor: https://192.168.5.173:8006/
- PVE-S500Plus:
icon: si-proxmox
href: https://192.168.5.174:8006/
- KubeVirt 虚拟化集群:
- KubeVirt-Shoryu:
icon: si-kubevirt
description: "CPU: R7-5825U / MEM: 64G / DISK: 1T"
siteMonitor: https://192.168.5.174:8006/
- PVE-GTR5:
icon: si-proxmox
href: https://192.168.5.172:8006/
href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.181:9100
siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.181:9100
- KubeVirt-Shushou:
icon: si-kubevirt
description: "CPU: R9-5900HX / MEM: 64G / DISK: 1T"
siteMonitor: https://192.168.5.172:8006/
href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.182:9100
siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.182:9100
- KubeVirt-Youko:
icon: si-kubevirt
description: "CPU: R5-5625U / MEM: 32G / DISK: 512G+4T*2"
href: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.183:9100
siteMonitor: http://grafana.writefor.fun/d/rYdddlPWk/node-exporter-full?orgId=1&var-node=192.168.5.183:9100
- LongHorn-Storage:
icon: si-longhorn
href: http://longhorn.writefor.fun/
- Victoria-Metrics:
icon: si-victoriametrics
href: http://vm.writefor.fun/
- KubeVirt-Grafana:
icon: si-grafana
href: http://k8s-grafana.writefor.fun/
- Homelab Monitoring:
- Grafana:

View File

@@ -40,8 +40,8 @@ useEqualHeights: true
# Groups and its layout
# Groups Name should match the name defined in your services.yaml or widgets.yaml
layout:
Proxmox VE 虚拟化集群:
icon: si-proxmox
KubeVirt 虚拟化集群:
icon: si-kubevirt
tab: First
Group A:

View File

@@ -8,7 +8,7 @@ in {
homelab-backup = {
inherit passwordFile;
initialize = true; # Initialize the repository if it doesn't exist.
repository = "rclone:smb-downloads:/Downloads/proxmox-backup/"; # backup to a rclone remote
repository = "rclone:smb-downloads:/Downloads/kubevirt-backup/"; # backup to a rclone remote
# rclone related
# rcloneOptions = {
@@ -34,20 +34,16 @@ in {
# A script that must run before starting the backup process.
backupPrepareCommand = ''
${pkgs.nushell}/bin/nu -c '
let pve_nodes = [
# proxmox cluster's nodes
"um560"
"gtr5"
"s500plus"
# others
"kana"
let kubevirt_nodes = [
"kubevirt-shoryu"
"kubevirt-shushou"
"kubevirt-youko"
]
pve_nodes | each {|it|
kubevirt_nodes | each {|it|
rsync -avz \
-e "ssh -i ${sshKeyPath}" \
$"($it):/var/lib/vz" $"/tmp/restic-backup-temp/($it)"
$"($it):/perissitent/" $"/tmp/restic-backup-temp/($it)"
}
'
'';

View File

@@ -1,7 +1,7 @@
{myvars, ...}:
#############################################################
#
# Aquamarine - A NixOS VM running on Proxmox
# Aquamarine - A NixOS VM running on Proxmox/KubeVirt
#
#############################################################
let

View File

@@ -5,7 +5,7 @@
}:
#############################################################
#
# Kana - a NixOS VM running on Proxmox
# Kana - a NixOS VM running on Proxmox/KubeVirt
#
#############################################################
let

View File

@@ -5,7 +5,7 @@
}:
#############################################################
#
# Ruby - a NixOS VM running on Proxmox
# Ruby - a NixOS VM running on Proxmox/KubeVirt
#
#############################################################
let

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-master-1"; # Define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-master-2"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-master-3"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-worker-1"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-worker-2"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-prod-1-worker-3"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-test-1-master-1"; # Define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-test-1-master-2"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,7 +7,7 @@
}: let
hostName = "k3s-test-1-master-3"; # define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtGuestModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -9,7 +9,7 @@
# MoreFine - S500Plus
hostName = "kubevirt-shoryu"; # Define your hostname.
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtHostModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,9 +7,8 @@
...
}: let
hostName = "kubevirt-shushou"; # Define your hostname.
k3sServerName = "kubevirt-shoryu";
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtHostModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -7,9 +7,8 @@
...
}: let
hostName = "kubevirt-youko"; # Define your hostname.
k3sServerName = "kubevirt-shoryu";
coreModule = mylib.genKubeVirtCoreModule {
coreModule = mylib.genKubeVirtHostModule {
inherit pkgs hostName;
inherit (myvars) networking;
};

View File

@@ -5,9 +5,10 @@
attrs = import ./attrs.nix {inherit lib;};
genKubeVirtCoreModule = import ./genKubeVirtCoreModule.nix;
genK3sServerModule = import ./genK3sServerModule.nix;
genK3sAgentModule = import ./genK3sAgentModule.nix;
genKubeVirtHostModule = import ./genKubeVirtHostModule.nix;
genKubeVirtGuestModule = import ./genKubeVirtGuestModule.nix;
# use path relative to the root of the project
relativeToRoot = lib.path.append ../.;

View File

@@ -0,0 +1,33 @@
{
pkgs,
hostName,
networking,
...
}: let
inherit (networking.hostsAddr.${hostName}) iface;
in {
# supported file systems, so we can mount any removable disks with these filesystems
boot.supportedFilesystems = [
"ext4"
"btrfs"
"xfs"
"fat"
"vfat"
"exfat"
];
networking = {
inherit hostName;
inherit (networking) defaultGateway nameservers;
inherit (networking.hostsInterface.${hostName}) interfaces;
networkmanager.enable = false;
};
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. Its perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "23.11"; # Did you read the comment?
}

View File

@@ -17,13 +17,6 @@ in
nixos-modules
++ [
nixos-generators.nixosModules.all-formats
{
# formatConfigs.iso = {config, ...}: {};
formatConfigs.proxmox = {config, ...}: {
# custom proxmox's image name
proxmox.qemuConf.name = "${config.networking.hostName}-nixos-${config.system.nixos.label}";
};
}
]
++ (
lib.optionals ((lib.lists.length home-modules) > 0)

View File

@@ -27,7 +27,8 @@
services.qemuGuest.enable = true;
services.openssh.enable = true;
services.cloud-init.enable = true;
# we configure the host via nixos itself, so we don't need the cloud-init
services.cloud-init.enable = false;
systemd.services."serial-getty@ttyS0".enable = true;
};
}

View File

@@ -1,36 +0,0 @@
{lib, ...}:
##############################################################################
#
# Template for Proxmox's VM, mainly based on:
# https://github.com/NixOS/nixpkgs/blob/nixos-unstable/nixos/modules/virtualisation/proxmox-image.nix
#
# the url above is used by `nixos-generator` to generate the Proxmox's VMA image file.
#
##############################################################################
{
boot = {
# after resize the disk, it will grow partition automatically.
growPartition = true;
kernelParams = ["console=ttyS0"];
loader.grub = {
device = "/dev/vda";
# we do not support EFI, so disable it.
efiSupport = false;
efiInstallAsRemovable = false;
};
loader.timeout = lib.mkForce 3; # wait for 3 seconds to select the boot entry
initrd.availableKernelModules = ["uas" "virtio_blk" "virtio_pci"];
};
fileSystems."/" = {
device = "/dev/disk/by-label/nixos";
autoResize = true;
fsType = "ext4";
};
# we do not have a /boot partition, so do not mount it.
# it alse had qemu-guest-agent installed by default.
services.qemuGuest.enable = lib.mkDefault true;
}

View File

@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -39,8 +39,7 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
# nixos tests
packages."${name}-nixos-tests" = import ../nixos-tests/idols-ruby.nix systemArgs;

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -39,6 +39,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -20,7 +20,7 @@
# common
"secrets/nixos.nix"
"modules/nixos/server/server.nix"
"modules/nixos/server/proxmox-hardware-configuration.nix"
"modules/nixos/server/kubevirt-hardware-configuration.nix"
# host specific
"hosts/k8s/${name}"
])
@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.proxmox;
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -39,6 +39,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -36,6 +36,5 @@ in {
colmena.${name} =
mylib.colmenaSystem (systemArgs // {inherit tags ssh-user;});
# generate proxmox image for virtual machines without desktop environment
packages.${name} = inputs.self.nixosConfigurations.${name}.config.formats.kubevirt;
}

View File

@@ -36,17 +36,17 @@
};
aquamarine = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.101";
};
ruby = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.102";
};
kana = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.103";
};
nozomi = {
@@ -61,7 +61,7 @@
};
chiaya = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.106";
};
suzu = {
@@ -90,32 +90,32 @@
# ============================================
k3s-prod-1-master-1 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.108";
};
k3s-prod-1-master-2 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.109";
};
k3s-prod-1-master-3 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.110";
};
k3s-prod-1-worker-1 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.111";
};
k3s-prod-1-worker-2 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.112";
};
k3s-prod-1-worker-3 = {
# VM
iface = "ens18";
iface = "eth1";
ipv4 = "192.168.5.113";
};